<commit_before><?xml version="1.0" ?>
<project default="default" name="VoltDB">

<!-- GENERAL HELPER MACROS -->
<macrodef name="envdefault">
    <attribute name="prop" />
    <attribute name="var" />
    <attribute name="default" />
    <sequential>
        <condition property="@{prop}" value="${env.@{var}}" else="@{default}">
            <isset property="env.@{var}" />
        </condition>
    </sequential>
</macrodef>

<macrodef name="invoke-javac">
    <attribute name="srcdir"/>
    <attribute name="destdir" default="${build.prod.dir}"/>
    <attribute name="excludes" default=""/>
    <sequential>
        <javac
            target="1.6"
            source="1.6"
            srcdir="@{srcdir}"
            destdir="@{destdir}"
            excludes="@{excludes}"
            encoding='UTF-8'
            debug='true'>
            <classpath refid="project.classpath" />
        </javac>
    </sequential>
</macrodef>

<!-- PATHS AND PROPERTIES -->

<tstamp/>

<!-- make environment var foo available as env.foo -->
<property environment="env"/>

<!-- allow env.VOLTBUILD to override "build" property -->
<envdefault prop="build" var="VOLTBUILD" default="release" />

<!-- allow env.VOLTPRO to override "voltpro" property -->
<condition property="voltpro" value="${env.VOLTPRO}">
    <isset property="env.VOLTPRO"/>
</condition>

<!-- stub the voltpro classpath and src.dir fileset so a fileset with this id always exists -->
<fileset id="voltpro.classpath" file="." excludes="**"/>
<fileset id="voltpro.src.dir" file="." excludes="**"/>

<!-- import the pro build.xml if it exists and is requested -->
<import file="${voltpro}/build.xml" optional="true"/>

<property name='base.dir'                   location='.' />
<property name='build.dir'                   location='obj/${build}' />
<property name='build.prod.dir'              location='${build.dir}/prod' />
<property name='build.test.dir'              location='${build.dir}/test' />
<property name='raw.dist.dir'                location='${build.dir}' />
<property name='dist.dir'                    location='${build.dir}/dist' />
<property name='dist.examples.dir'           location='${dist.dir}/examples' />
<property name='doc.dir'                     location='doc' />
<property name='src.gpl.dir'                 location='src/frontend' />
<property name='src.hsqldb.dir'              location='src/hsqldb19b3' />
<property name='src.test.dir'                location='tests/frontend' />
<property name='src.hsqldb.test.dir'         location='tests/hsqldb' />
<property name='src.studio.dir'              location='src/frontend/org/voltdb/studio' />
<property name='build.testoutput.dir'        location='${build.dir}/testoutput' />
<property name='build.testobjects.dir'       location='${build.dir}/testobjects' />
<property name='lib.dir'                     location='lib' />
<property name='vendor.lib.dir'              location='third_party/java/jars' />
<property name='vendor.src.dir'              location='third_party/java/src'  />
<property name='src.ee.parent.dir'           location='src/ee' />
<property name='src.ee.dir'                  location='src/ee' />
<property name='src.catalog.dir'             location='src/catgen' />
<property name='depcache'                    value='.depcache' />

<!-- emma build instrumentation location -->
<property name='build.instr.dir'             location='${build.dir}/instr' />

<!-- Default heap size for Volt server (MB)  -->
<condition property="volt.server.memory" value="2048">
    <not><isset property="volt.server.memory"/></not>
</condition>

<!-- Default heap size for Volt clients+loaders (MB)  -->
<condition property="volt.client.memory" value="2048">
    <not><isset property="volt.client.memory"/></not>
</condition>

<!-- Overridden in the Hudson test script. -->
<property name='junit.haltonfailure'    value='false' />
<property name="j2se_api" value="http://download.oracle.com/javase/6/docs/api/"/>

<path id='project.classpath'>
    <pathelement location='${build.instr.dir}' />
    <pathelement location='${build.prod.dir}' />
    <pathelement location='${build.test.dir}' />
    <fileset dir='${lib.dir}'>
        <include name='*.jar' />
    </fileset>
    <fileset dir='${vendor.lib.dir}'>
        <include name='*.jar' />
        <exclude name='ant.jar' />
    </fileset>
    <pathelement path="${java.class.path}"/>
    <fileset refid="voltpro.classpath"/>
</path>

<!-- Workload Tracer Properties -->
<condition property="workload.trace.class" value="">
    <not><isset property="workload.trace.class"/></not>
</condition>
<condition property="workload.trace.path" value="">
    <not><isset property="workload.trace.path"/></not>
</condition>
<condition property="workload.trace.ignore" value="">
    <not><isset property="workload.trace.ignore"/></not>
</condition>

<!--
***************************************
PRIMARY ENTRY POINTS
***************************************
-->

<target name="default"
    depends="compile, ee, voltdb.jar"
    description="Compile Java classes and C++ JNI library."
/>
<target name="check"
    depends="licensecheck, compile, voltdbipc, ee"
    description="Run Java and C++ JNI testcases." >
    <condition property="timeoutLength" value="${timeoutLength}" else='1800000'>
        <isset property="timeoutLength"/>
    </condition>

    <exec executable="/usr/bin/ant" failonerror="true" timeout="9000000">
        <arg value="eecheck" />
        <arg value="junit" />
        <arg value="distcheck" />
        <arg value="pythonfser" />
        <arg value="-Dbuild=${build}" />
        <arg value="-DtimeoutLength=${timeoutLength}" />
        <arg value="-DVOLT_REGRESSIONS=${VOLT_REGRESSIONS}" />
    </exec>
</target>
<target name="check_noclustering"
    depends="licensecheck, compile, voltdbipc, ee"
    description="Run Java and C++ JNI testcases that apply to a single node." >
    <condition property="timeoutLength" value="${timeoutLength}" else='1800000'>
        <isset property="timeoutLength"/>
    </condition>

    <exec executable="/usr/bin/ant" failonerror="true" timeout="9000000">
        <arg value="eecheck" />
        <arg value="junit_noclustering" />
        <arg value="distcheck" />
        <arg value="pythonfser" />
        <arg value="-Dbuild=${build}" />
        <arg value="-DtimeoutLength=${timeoutLength}" />
        <arg value="-DVOLT_REGRESSIONS=localhsql" />
    </exec>
</target>
<target name="check_quick"
    depends="compile, voltdbipc"
    description="Run a subset of Java testcases and test fragments." >
    <condition property="timeoutLength" value="${timeoutLength}" else='480000'>
        <isset property="timeoutLength"/>
    </condition>

    <exec executable="/usr/bin/ant" failonerror="true" timeout="1800000">
        <arg value="licensecheck" />
        <arg value="junit_quick" />
        <arg value="distcheck" />
        <arg value="-Dbuild=${build}" />
        <arg value="-DtimeoutLength=${timeoutLength}" />
        <arg value="-DVOLT_REGRESSIONS=${VOLT_REGRESSIONS}" />
    </exec>
</target>

<target name="quarantine"
    depends="compile, ee"
    description="Run quarantined tests." >
    <condition property="timeoutLength" value="${timeoutLength}" else='480000'>
        <isset property="timeoutLength"/>
    </condition>

    <exec executable="/usr/bin/ant" failonerror="true" timeout="1800000">
        <arg value="junit_quarantine" />
        <arg value="-Dbuild=${build}" />
        <arg value="-DtimeoutLength=${timeoutLength}" />
        <arg value="-DVOLT_REGRESSIONS=${VOLT_REGRESSIONS}" />
    </exec>
</target>

<target name="killstragglers"
        description="Run the killstragglers script">
    <exec executable="tools/killstragglers.sh"
          failonerror="true" timeout="1800000">
    </exec>
</target>

<target name="all"
    depends="compile, ee, junit, eecheck, javadoc, jars"
    description="Do all tasks."
/>
<target name="jars"
    depends="voltdb.jar, voltdbfat.jar, voltdbclient.jar"
    description="Create production JAR files."
/>
<target name="dist"
    depends="dist_client, dist_internal, dist_voltcache, dist_voltkv"
    description="Create VoltDB release packages with examples and documentation."
/>
<target name="dist_voltcache"  depends="dist_internal"
        description="Create VoltDB release package with VoltCache reference app.">
  <refapp_macro appname="voltcache" />
</target>

<target name="dist_voltkv" depends="dist_internal"
        description="Create VoltDB release package with VoltKV reference app.">
  <refapp_macro appname="voltkv" />
</target>

<target name="dist_refapp" depends="dist_internal"
        description="Create reference application release package with app supplied in ant invocation.">
  <fail unless="appname" message="must specify an app name with -Dappname=" />
  <refapp_macro appname="${appname}" />
</target>

<!--
***************************************
DISTRIBUTION
***************************************
-->

<macrodef name="voltbin-macro">
  <sequential>
    <mkdir dir="${build.dir}/voltbin" />
    <copy todir="${build.dir}/voltbin" flatten="true">
        <fileset dir="${build.dir}" defaultexcludes="yes">
            <include name="nativelibs/libvoltdb*" />
            <include name="prod/voltdbfat.jar" />
        </fileset>
    </copy>
    <!-- use cp to preserve permissions -->
    <exec dir="tools" executable="cp" failonerror="true">
        <arg line="getmac.py"/>
        <arg line="killstragglers.sh"/>
        <arg line="${build.dir}/voltbin"/>
    </exec>
    <!-- populate the logging configuration files -->
    <copy tofile="${build.dir}/voltbin/log4j.properties" file="voltdb/log4j.properties"/>
    <copy tofile="${build.dir}/voltbin/log4j.xml" file="voltdb/log4j.xml"/>
  </sequential>
</macrodef>

<target name="voltbin" depends="compile, ee, jars">
  <voltbin-macro/>

  <!-- strip the voltbin shared library (~40x size reduction) -->
  <exec dir='${build.dir}/voltbin' executable='/bin/sh'>
    <arg line="-c '/usr/bin/strip -x libvoltdb*'"/>
  </exec>
</target>

<target name="voltbin_profile" depends="compile, ee_profile, jars">
  <voltbin-macro/>
</target>

<target name="voltbin_copy" depends="compile,ee,jars">
  <voltbin-macro/>

  <!-- copy the voltbin contents to $HOME/voltbin -->
  <exec dir="${build.dir}" executable='/bin/sh'>
    <arg line="-c 'cp voltbin/* $HOME/voltbin/'"/>
  </exec>
</target>

<target name="javadoc">
    <!-- populate selected server/compiler javadoc documentation -->
    <javadoc
        destdir="doc/javadoc/procedure-api"
        Public="true"
        version="true"
        use="true"
        nodeprecated="true"
        Overview='${src.gpl.dir}/overview-public.html'
        Windowtitle='VoltDB Server APIs'>
        <link href="${j2se_api}"/>
        <classpath refid='project.classpath' />
        <fileset dir="." defaultexcludes="yes">
            <include name="src/frontend/org/voltdb/VoltTable.java" />
            <include name="src/frontend/org/voltdb/VoltTableRow.java" />
            <include name="src/frontend/org/voltdb/VoltProcedure.java" />
            <include name="src/frontend/org/voltdb/SQLStmt.java" />
            <include name="src/frontend/org/voltdb/VoltType.java" />
            <include name="src/frontend/org/voltdb/ProcInfo.java" />
            <include name="src/frontend/org/voltdb/types/TimestampType.java" />
        </fileset>
    </javadoc>

    <!-- populate selected client javadoc documentation -->
    <javadoc
        destdir="doc/javadoc/java-client-api"
        access="protected"
        version="true"
        use="true"
        nodeprecated="true"
        Overview='${src.gpl.dir}/overview-public.html'
        Windowtitle='VoltDB Client APIs'>
        <link href="${j2se_api}"/>
        <classpath refid='project.classpath' />
        <fileset dir="." defaultexcludes="yes">
            <include name="src/frontend/org/voltdb/VoltTable.java" />
            <include name="src/frontend/org/voltdb/VoltTableRow.java" />
            <include name="src/frontend/org/voltdb/VoltClient.java" />
            <include name="src/frontend/org/voltdb/VoltType.java" />
            <include name="src/frontend/org/voltdb/client/Client.java" />
            <include name="src/frontend/org/voltdb/client/NoConnectionsException.java" />
            <include name="src/frontend/org/voltdb/client/ProcedureCallback.java" />
            <include name="src/frontend/org/voltdb/client/ClientConfig.java" />
            <include name="src/frontend/org/voltdb/client/ClientFactory.java" />
            <include name="src/frontend/org/voltdb/client/SyncCallback.java" />
            <include name="src/frontend/org/voltdb/client/NullCallback.java" />
            <include name="src/frontend/org/voltdb/client/ProcCallException.java" />
            <include name="src/frontend/org/voltdb/client/ClientStatusListener.java" />
            <include name="src/frontend/org/voltdb/client/ClientStatusListenerExt.java" />
            <include name="src/frontend/org/voltdb/client/ClientResponse.java" />
            <include name="src/frontend/org/voltdb/client/ClientStats.java" />
            <include name="src/frontend/org/voltdb/client/ClientStatsContext.java" />
            <include name="src/frontend/org/voltdb/types/TimestampType.java" />
        </fileset>
    </javadoc>
</target>

<target name="dist_internal" depends="compile, ee, javadoc, voltdb.jar, voltdbclient.jar">
    <!-- prepare release directory for new content -->
    <delete includeemptydirs="true" failonerror='false'>
        <fileset dir="${dist.dir}" includes="**/*" />
    </delete>
    <mkdir dir="${dist.dir}" />

    <!-- populate the doc dir -->
    <copy todir="${dist.dir}/">
        <fileset dir="." defaultexcludes="yes">
            <include name="Click Here to Start.html"/>
            <include name="README"/>
            <include name="README.thirdparty"/>
            <include name="LICENSE"/>
            <include name="bin/**"/>
            <include name="doc/**"/>
            <include name="examples/**"/>
            <include name="lib/**"/>
            <include name="tools/demo/**"/>
            <include name="tools/sqlcmd/**"/>
            <include name="voltdb/**"/>
            <exclude name=".gitignore"/>
        </fileset>
    </copy>

    <antcall target="copy_pro_bin"/>

    <!-- populate java and native libraries -->
    <copy todir="${dist.dir}/voltdb" flatten="true" >
        <fileset dir="voltdb" defaultexcludes="yes">
            <include name="voltdb-${dist.version}.jar" />
            <include name="voltdb-${dist.version}-javadoc.jar" />
            <include name="voltdbclient-${dist.version}.jar" />
            <include name="voltdbclient-${dist.version}-javadoc.jar" />
        </fileset>
    </copy>
    <copy todir="${dist.dir}/voltdb" flatten="true" >
        <fileset dir="${build.dir}" defaultexcludes="yes">
            <include name="nativelibs/libvoltdb*" />
        </fileset>
    </copy>
    <copy todir="${dist.dir}/voltdb" flatten="true" >
        <fileset dir="." defaultexcludes="yes">
            <include name="third_party/cpp/jnilib/libjzmq*" />
        </fileset>
    </copy>

    <!-- save the symbols from the shared library -->
    <exec dir='${dist.dir}/voltdb' executable='/bin/sh'>
        <arg line="-c '/usr/bin/objcopy --only-keep-debug libvoltdb-${dist.version}.so ${build.dir}/voltdb-${dist.version}.sym'" />
    </exec>

    <!-- strip the voltbin shared library (~40x size reduction) -->
    <exec dir='${dist.dir}/voltdb' executable='/bin/sh'>
        <arg line="-c '/usr/bin/strip --strip-debug --strip-unneeded libvoltdb*'"/>
    </exec>

    <!-- embed the path to the symbol file into the ELF binary -->
    <exec dir='${dist.dir}/voltdb' executable='/bin/sh'>
        <arg line="-c '/usr/bin/objcopy --add-gnu-debuglink=voltdb-${dist.version}.sym libvoltdb-${dist.version}.so'" />
    </exec>

    <!-- copy studio.web -->
    <mkdir dir="${dist.dir}/tools/studio.web" />
    <copy todir="${dist.dir}/tools/studio.web">
        <fileset dir="${src.gpl.dir}/org/voltdb/studio" defaultexcludes="yes">
            <include name="**"/>
        </fileset>
    </copy>

    <!-- populate project generator -->
    <exec dir="src/proj_gen/" executable="python">
        <arg line="generator_compiler.py"/>
    </exec>
    <move tofile="${dist.dir}/tools/generate" file="src/proj_gen/generate" />

    <!-- add the Project and Deployment file schemas to the dist -->
    <copy todir="${dist.dir}/tools" file="src/frontend/org/voltdb/compiler/ProjectFileSchema.xsd"/>
    <copy todir="${dist.dir}/tools" file="src/frontend/org/voltdb/compiler/DeploymentFileSchema.xsd"/>

    <!-- copy license to voltdb dir -->
    <copy todir="${dist.dir}/voltdb" file="LICENSE"/>

    <!-- make shell scripts executable -->
    <chmod perm="ugo+rx">
        <fileset dir="${dist.dir}" defaultexcludes="yes">
            <include name="bin/*"/>
            <include name="tools/demo/launch"/>
            <include name="doc/tutorials/**/run.sh"/>
            <include name="examples/**/run.sh"/>
            <include name="tools/generate"/>
            <include name="tools/sqlcmd/sqlcmd"/>
        </fileset>
    </chmod>

    <!-- create an archive for distribution -->
    <exec executable="mv" failonerror="true">
        <arg value="${raw.dist.dir}/dist"/>
        <arg value="${raw.dist.dir}/voltdb-${dist.version}"/>
    </exec>
    <exec executable="tar" failonerror="true">
        <arg value="-cz"/>
        <arg value="-C"/>
        <arg value="${raw.dist.dir}"/>
        <arg value="-f"/>
        <arg value="${raw.dist.dir}/voltdb-${dist.version}.tar.gz"/>
        <arg value="voltdb-${dist.version}"/>
    </exec>
    <!-- move it back to dist directory for downstream dependencies -->
    <exec executable="mv" failonerror="true">
        <arg value="${raw.dist.dir}/voltdb-${dist.version}"/>
        <arg value="${raw.dist.dir}/dist"/>
    </exec>

    <!-- package up studio web trivially -->
    <zip destfile="${raw.dist.dir}/voltdb-studio.web-${dist.version}.zip">
        <fileset dir="${src.studio.dir}" defaultexcludes="yes">
            <include name="**/*" />
        </fileset>
    </zip>
</target>

<target name="dist_client" depends="dist_internal"
        description="Hacky Java client package target">
    <!-- prepare release directory for new content -->
    <delete includeemptydirs="true" failonerror='false'>
        <fileset dir="${dist.dir}-client-java" includes="**/*" />
    </delete>
    <mkdir dir="${dist.dir}-client-java" />

    <!-- populate the dist-client-java dir from the superset dist -->
    <copy todir="${dist.dir}-client-java/">
        <fileset dir="${dist.dir}/" defaultexcludes="yes">
            <include name="**/*"/>
            <exclude name=".gitignore"/>
            <exclude name="bin/"/>
            <exclude name="voltdb/libvoltdb*"/>
            <exclude name="voltdb/voltdb-*.jar"/>
            <exclude name="voltdb/libjzmq*"/>
        </fileset>
    </copy>

    <!-- make shell scripts executable -->
    <chmod perm="ugo+rx">
        <fileset dir="${dist.dir}-client-java/" defaultexcludes="yes">
            <include name="tools/demo/launch"/>
            <include name="doc/tutorials/auction/run.sh"/>
            <include name="doc/tutorials/helloworld/run.sh"/>
            <include name="examples/voltcache/run.sh"/>
            <include name="examples/voltkv/run.sh"/>
            <include name="examples/voter/run.sh"/>
            <include name="tools/generate"/>
            <include name="tools/sqlcmd/sqlcmd"/>
        </fileset>
    </chmod>

    <!-- create an archive for distribution -->
    <exec executable="mv" failonerror="true">
        <arg value="${raw.dist.dir}/dist-client-java"/>
        <arg value="${raw.dist.dir}/voltdb-client-java-${dist.version}"/>
    </exec>
    <exec executable="tar" failonerror="true">
        <arg value="-cz"/>
        <arg value="-C"/>
        <arg value="${raw.dist.dir}"/>
        <arg value="-f"/>
        <arg value="${raw.dist.dir}/voltdb-client-java-${dist.version}.tar.gz"/>
        <arg value="voltdb-client-java-${dist.version}"/>
    </exec>
    <!-- move it back to dist directory for downstream dependencies -->
    <exec executable="mv" failonerror="true">
        <arg value="${raw.dist.dir}/voltdb-client-java-${dist.version}"/>
        <arg value="${raw.dist.dir}/dist-client-java"/>
    </exec>
</target>

<macrodef name="refapp_macro"
        description="Build a kit that only includes a single reference app">
  <attribute name="appname"/>
  <sequential>

    <!-- prepare release directory for new content -->
    <delete includeemptydirs="true" failonerror='false'>
        <fileset dir="${dist.dir}-@{appname}" includes="**/*" />
    </delete>
    <mkdir dir="${dist.dir}-@{appname}" />
    <mkdir dir="${dist.dir}-@{appname}/@{appname}" />

    <!-- populate the examples -->
    <copy todir="${dist.dir}-@{appname}/@{appname}">
        <fileset dir="${dist.dir}/examples/@{appname}" defaultexcludes="yes">
            <include name="**/*"/>
            <exclude name=".gitignore"/>
        </fileset>
    </copy>

    <!-- populate the bin/lib/voltdb/etc... dirs -->
    <copy todir="${dist.dir}-@{appname}" >
        <fileset dir="${dist.dir}" defaultexcludes="yes">
            <include name="README" />
            <include name="LICENSE" />
            <include name="README.thirdparty" />
            <include name="voltdb/**" />
            <include name="lib/**" />
            <include name="bin/**" />
        </fileset>
    </copy>

    <!-- rewrite the run.sh script's relative directories -->
    <replace
        file="${dist.dir}-@{appname}/@{appname}/run.sh"
        token="../../"
        value="../"/>

    <!-- make shell scripts executable -->
    <chmod perm="ugo+rx">
        <fileset dir="${dist.dir}-@{appname}" defaultexcludes="yes">
            <include name="bin/*"/>
            <include name="@{appname}/run.sh"/>
        </fileset>
    </chmod>



    <!-- create an archive for distribution -->
    <exec executable="mv" failonerror="true">
        <arg value="${raw.dist.dir}/dist-@{appname}"/>
        <arg value="${raw.dist.dir}/voltdb-@{appname}-${dist.version}"/>
    </exec>
    <exec executable="tar" failonerror="true">
        <arg value="-cz"/>
        <arg value="-C"/>
        <arg value="${raw.dist.dir}"/>
        <arg value="-f"/>
        <arg value="${raw.dist.dir}/voltdb-@{appname}-${dist.version}.tar.gz"/>
        <arg value="voltdb-@{appname}-${dist.version}"/>
    </exec>
    <!-- move it back to dist directory for downstream dependencies -->
    <exec executable="mv" failonerror="true">
        <arg value="${raw.dist.dir}/voltdb-@{appname}-${dist.version}"/>
        <arg value="${raw.dist.dir}/dist-@{appname}"/>
    </exec>
  </sequential>
</macrodef>

<!--
***************************************
CLEANING
***************************************
-->

<target name='clean' description="Remove all compiled files." depends="cleantmp">
  <exec dir='examples/voter' executable='/usr/bin/env'><arg line="bash run.sh clean"/></exec>
  <exec dir='examples/voltcache' executable='/usr/bin/env'><arg line="bash run.sh clean"/></exec>
  <exec dir='examples/voltkv' executable='/usr/bin/env'><arg line="bash run.sh clean"/></exec>
  <exec dir='doc/tutorials/auction' executable='/usr/bin/env'><arg line="bash run.sh clean"/></exec>
  <exec dir='doc/tutorials/helloworld' executable='/usr/bin/env'><arg line="bash run.sh clean"/></exec>  <exec dir='.' executable='/bin/sh'>
    <arg line="-c 'rm -rf obj/*'"/>
  </exec>
  <exec dir='.' executable='/bin/sh'>
    <arg line="-c 'rm -rf voltdb/*.jar'"/>
  </exec>
  <exec dir='.' executable='/bin/sh'>
    <arg line="-c 'rm -rf voltdb/*.so'"/>
  </exec>
  <exec dir='.' executable='/bin/sh'>
    <arg line="-c 'rm -rf src/ee/catalog/*'"/>
  </exec>
  <exec dir='.' executable='/bin/sh'>
    <arg line="-c 'rm -rf src/frontend/org/voltdb/catalog/*.java'"/><commit_msg>Make "ant clean" remove the mac artifacts that end in .jnilib<commit_after><?xml version="1.0" ?>
<project default="default" name="VoltDB">

<!-- GENERAL HELPER MACROS -->
<macrodef name="envdefault">
    <attribute name="prop" />
    <attribute name="var" />
    <attribute name="default" />
    <sequential>
        <condition property="@{prop}" value="${env.@{var}}" else="@{default}">
            <isset property="env.@{var}" />
        </condition>
    </sequential>
</macrodef>

<macrodef name="invoke-javac">
    <attribute name="srcdir"/>
    <attribute name="destdir" default="${build.prod.dir}"/>
    <attribute name="excludes" default=""/>
    <sequential>
        <javac
            target="1.6"
            source="1.6"
            srcdir="@{srcdir}"
            destdir="@{destdir}"
            excludes="@{excludes}"
            encoding='UTF-8'
            debug='true'>
            <classpath refid="project.classpath" />
        </javac>
    </sequential>
</macrodef>

<!-- PATHS AND PROPERTIES -->

<tstamp/>

<!-- make environment var foo available as env.foo -->
<property environment="env"/>

<!-- allow env.VOLTBUILD to override "build" property -->
<envdefault prop="build" var="VOLTBUILD" default="release" />

<!-- allow env.VOLTPRO to override "voltpro" property -->
<condition property="voltpro" value="${env.VOLTPRO}">
    <isset property="env.VOLTPRO"/>
</condition>

<!-- stub the voltpro classpath and src.dir fileset so a fileset with this id always exists -->
<fileset id="voltpro.classpath" file="." excludes="**"/>
<fileset id="voltpro.src.dir" file="." excludes="**"/>

<!-- import the pro build.xml if it exists and is requested -->
<import file="${voltpro}/build.xml" optional="true"/>

<property name='base.dir'                   location='.' />
<property name='build.dir'                   location='obj/${build}' />
<property name='build.prod.dir'              location='${build.dir}/prod' />
<property name='build.test.dir'              location='${build.dir}/test' />
<property name='raw.dist.dir'                location='${build.dir}' />
<property name='dist.dir'                    location='${build.dir}/dist' />
<property name='dist.examples.dir'           location='${dist.dir}/examples' />
<property name='doc.dir'                     location='doc' />
<property name='src.gpl.dir'                 location='src/frontend' />
<property name='src.hsqldb.dir'              location='src/hsqldb19b3' />
<property name='src.test.dir'                location='tests/frontend' />
<property name='src.hsqldb.test.dir'         location='tests/hsqldb' />
<property name='src.studio.dir'              location='src/frontend/org/voltdb/studio' />
<property name='build.testoutput.dir'        location='${build.dir}/testoutput' />
<property name='build.testobjects.dir'       location='${build.dir}/testobjects' />
<property name='lib.dir'                     location='lib' />
<property name='vendor.lib.dir'              location='third_party/java/jars' />
<property name='vendor.src.dir'              location='third_party/java/src'  />
<property name='src.ee.parent.dir'           location='src/ee' />
<property name='src.ee.dir'                  location='src/ee' />
<property name='src.catalog.dir'             location='src/catgen' />
<property name='depcache'                    value='.depcache' />

<!-- emma build instrumentation location -->
<property name='build.instr.dir'             location='${build.dir}/instr' />

<!-- Default heap size for Volt server (MB)  -->
<condition property="volt.server.memory" value="2048">
    <not><isset property="volt.server.memory"/></not>
</condition>

<!-- Default heap size for Volt clients+loaders (MB)  -->
<condition property="volt.client.memory" value="2048">
    <not><isset property="volt.client.memory"/></not>
</condition>

<!-- Overridden in the Hudson test script. -->
<property name='junit.haltonfailure'    value='false' />
<property name="j2se_api" value="http://download.oracle.com/javase/6/docs/api/"/>

<path id='project.classpath'>
    <pathelement location='${build.instr.dir}' />
    <pathelement location='${build.prod.dir}' />
    <pathelement location='${build.test.dir}' />
    <fileset dir='${lib.dir}'>
        <include name='*.jar' />
    </fileset>
    <fileset dir='${vendor.lib.dir}'>
        <include name='*.jar' />
        <exclude name='ant.jar' />
    </fileset>
    <pathelement path="${java.class.path}"/>
    <fileset refid="voltpro.classpath"/>
</path>

<!-- Workload Tracer Properties -->
<condition property="workload.trace.class" value="">
    <not><isset property="workload.trace.class"/></not>
</condition>
<condition property="workload.trace.path" value="">
    <not><isset property="workload.trace.path"/></not>
</condition>
<condition property="workload.trace.ignore" value="">
    <not><isset property="workload.trace.ignore"/></not>
</condition>

<!--
***************************************
PRIMARY ENTRY POINTS
***************************************
-->

<target name="default"
    depends="compile, ee, voltdb.jar"
    description="Compile Java classes and C++ JNI library."
/>
<target name="check"
    depends="licensecheck, compile, voltdbipc, ee"
    description="Run Java and C++ JNI testcases." >
    <condition property="timeoutLength" value="${timeoutLength}" else='1800000'>
        <isset property="timeoutLength"/>
    </condition>

    <exec executable="/usr/bin/ant" failonerror="true" timeout="9000000">
        <arg value="eecheck" />
        <arg value="junit" />
        <arg value="distcheck" />
        <arg value="pythonfser" />
        <arg value="-Dbuild=${build}" />
        <arg value="-DtimeoutLength=${timeoutLength}" />
        <arg value="-DVOLT_REGRESSIONS=${VOLT_REGRESSIONS}" />
    </exec>
</target>
<target name="check_noclustering"
    depends="licensecheck, compile, voltdbipc, ee"
    description="Run Java and C++ JNI testcases that apply to a single node." >
    <condition property="timeoutLength" value="${timeoutLength}" else='1800000'>
        <isset property="timeoutLength"/>
    </condition>

    <exec executable="/usr/bin/ant" failonerror="true" timeout="9000000">
        <arg value="eecheck" />
        <arg value="junit_noclustering" />
        <arg value="distcheck" />
        <arg value="pythonfser" />
        <arg value="-Dbuild=${build}" />
        <arg value="-DtimeoutLength=${timeoutLength}" />
        <arg value="-DVOLT_REGRESSIONS=localhsql" />
    </exec>
</target>
<target name="check_quick"
    depends="compile, voltdbipc"
    description="Run a subset of Java testcases and test fragments." >
    <condition property="timeoutLength" value="${timeoutLength}" else='480000'>
        <isset property="timeoutLength"/>
    </condition>

    <exec executable="/usr/bin/ant" failonerror="true" timeout="1800000">
        <arg value="licensecheck" />
        <arg value="junit_quick" />
        <arg value="distcheck" />
        <arg value="-Dbuild=${build}" />
        <arg value="-DtimeoutLength=${timeoutLength}" />
        <arg value="-DVOLT_REGRESSIONS=${VOLT_REGRESSIONS}" />
    </exec>
</target>

<target name="quarantine"
    depends="compile, ee"
    description="Run quarantined tests." >
    <condition property="timeoutLength" value="${timeoutLength}" else='480000'>
        <isset property="timeoutLength"/>
    </condition>

    <exec executable="/usr/bin/ant" failonerror="true" timeout="1800000">
        <arg value="junit_quarantine" />
        <arg value="-Dbuild=${build}" />
        <arg value="-DtimeoutLength=${timeoutLength}" />
        <arg value="-DVOLT_REGRESSIONS=${VOLT_REGRESSIONS}" />
    </exec>
</target>

<target name="killstragglers"
        description="Run the killstragglers script">
    <exec executable="tools/killstragglers.sh"
          failonerror="true" timeout="1800000">
    </exec>
</target>

<target name="all"
    depends="compile, ee, junit, eecheck, javadoc, jars"
    description="Do all tasks."
/>
<target name="jars"
    depends="voltdb.jar, voltdbfat.jar, voltdbclient.jar"
    description="Create production JAR files."
/>
<target name="dist"
    depends="dist_client, dist_internal, dist_voltcache, dist_voltkv"
    description="Create VoltDB release packages with examples and documentation."
/>
<target name="dist_voltcache"  depends="dist_internal"
        description="Create VoltDB release package with VoltCache reference app.">
  <refapp_macro appname="voltcache" />
</target>

<target name="dist_voltkv" depends="dist_internal"
        description="Create VoltDB release package with VoltKV reference app.">
  <refapp_macro appname="voltkv" />
</target>

<target name="dist_refapp" depends="dist_internal"
        description="Create reference application release package with app supplied in ant invocation.">
  <fail unless="appname" message="must specify an app name with -Dappname=" />
  <refapp_macro appname="${appname}" />
</target>

<!--
***************************************
DISTRIBUTION
***************************************
-->

<macrodef name="voltbin-macro">
  <sequential>
    <mkdir dir="${build.dir}/voltbin" />
    <copy todir="${build.dir}/voltbin" flatten="true">
        <fileset dir="${build.dir}" defaultexcludes="yes">
            <include name="nativelibs/libvoltdb*" />
            <include name="prod/voltdbfat.jar" />
        </fileset>
    </copy>
    <!-- use cp to preserve permissions -->
    <exec dir="tools" executable="cp" failonerror="true">
        <arg line="getmac.py"/>
        <arg line="killstragglers.sh"/>
        <arg line="${build.dir}/voltbin"/>
    </exec>
    <!-- populate the logging configuration files -->
    <copy tofile="${build.dir}/voltbin/log4j.properties" file="voltdb/log4j.properties"/>
    <copy tofile="${build.dir}/voltbin/log4j.xml" file="voltdb/log4j.xml"/>
  </sequential>
</macrodef>

<target name="voltbin" depends="compile, ee, jars">
  <voltbin-macro/>

  <!-- strip the voltbin shared library (~40x size reduction) -->
  <exec dir='${build.dir}/voltbin' executable='/bin/sh'>
    <arg line="-c '/usr/bin/strip -x libvoltdb*'"/>
  </exec>
</target>

<target name="voltbin_profile" depends="compile, ee_profile, jars">
  <voltbin-macro/>
</target>

<target name="voltbin_copy" depends="compile,ee,jars">
  <voltbin-macro/>

  <!-- copy the voltbin contents to $HOME/voltbin -->
  <exec dir="${build.dir}" executable='/bin/sh'>
    <arg line="-c 'cp voltbin/* $HOME/voltbin/'"/>
  </exec>
</target>

<target name="javadoc">
    <!-- populate selected server/compiler javadoc documentation -->
    <javadoc
        destdir="doc/javadoc/procedure-api"
        Public="true"
        version="true"
        use="true"
        nodeprecated="true"
        Overview='${src.gpl.dir}/overview-public.html'
        Windowtitle='VoltDB Server APIs'>
        <link href="${j2se_api}"/>
        <classpath refid='project.classpath' />
        <fileset dir="." defaultexcludes="yes">
            <include name="src/frontend/org/voltdb/VoltTable.java" />
            <include name="src/frontend/org/voltdb/VoltTableRow.java" />
            <include name="src/frontend/org/voltdb/VoltProcedure.java" />
            <include name="src/frontend/org/voltdb/SQLStmt.java" />
            <include name="src/frontend/org/voltdb/VoltType.java" />
            <include name="src/frontend/org/voltdb/ProcInfo.java" />
            <include name="src/frontend/org/voltdb/types/TimestampType.java" />
        </fileset>
    </javadoc>

    <!-- populate selected client javadoc documentation -->
    <javadoc
        destdir="doc/javadoc/java-client-api"
        access="protected"
        version="true"
        use="true"
        nodeprecated="true"
        Overview='${src.gpl.dir}/overview-public.html'
        Windowtitle='VoltDB Client APIs'>
        <link href="${j2se_api}"/>
        <classpath refid='project.classpath' />
        <fileset dir="." defaultexcludes="yes">
            <include name="src/frontend/org/voltdb/VoltTable.java" />
            <include name="src/frontend/org/voltdb/VoltTableRow.java" />
            <include name="src/frontend/org/voltdb/VoltClient.java" />
            <include name="src/frontend/org/voltdb/VoltType.java" />
            <include name="src/frontend/org/voltdb/client/Client.java" />
            <include name="src/frontend/org/voltdb/client/NoConnectionsException.java" />
            <include name="src/frontend/org/voltdb/client/ProcedureCallback.java" />
            <include name="src/frontend/org/voltdb/client/ClientConfig.java" />
            <include name="src/frontend/org/voltdb/client/ClientFactory.java" />
            <include name="src/frontend/org/voltdb/client/SyncCallback.java" />
            <include name="src/frontend/org/voltdb/client/NullCallback.java" />
            <include name="src/frontend/org/voltdb/client/ProcCallException.java" />
            <include name="src/frontend/org/voltdb/client/ClientStatusListener.java" />
            <include name="src/frontend/org/voltdb/client/ClientStatusListenerExt.java" />
            <include name="src/frontend/org/voltdb/client/ClientResponse.java" />
            <include name="src/frontend/org/voltdb/client/ClientStats.java" />
            <include name="src/frontend/org/voltdb/client/ClientStatsContext.java" />
            <include name="src/frontend/org/voltdb/types/TimestampType.java" />
        </fileset>
    </javadoc>
</target>

<target name="dist_internal" depends="compile, ee, javadoc, voltdb.jar, voltdbclient.jar">
    <!-- prepare release directory for new content -->
    <delete includeemptydirs="true" failonerror='false'>
        <fileset dir="${dist.dir}" includes="**/*" />
    </delete>
    <mkdir dir="${dist.dir}" />

    <!-- populate the doc dir -->
    <copy todir="${dist.dir}/">
        <fileset dir="." defaultexcludes="yes">
            <include name="Click Here to Start.html"/>
            <include name="README"/>
            <include name="README.thirdparty"/>
            <include name="LICENSE"/>
            <include name="bin/**"/>
            <include name="doc/**"/>
            <include name="examples/**"/>
            <include name="lib/**"/>
            <include name="tools/demo/**"/>
            <include name="tools/sqlcmd/**"/>
            <include name="voltdb/**"/>
            <exclude name=".gitignore"/>
        </fileset>
    </copy>

    <antcall target="copy_pro_bin"/>

    <!-- populate java and native libraries -->
    <copy todir="${dist.dir}/voltdb" flatten="true" >
        <fileset dir="voltdb" defaultexcludes="yes">
            <include name="voltdb-${dist.version}.jar" />
            <include name="voltdb-${dist.version}-javadoc.jar" />
            <include name="voltdbclient-${dist.version}.jar" />
            <include name="voltdbclient-${dist.version}-javadoc.jar" />
        </fileset>
    </copy>
    <copy todir="${dist.dir}/voltdb" flatten="true" >
        <fileset dir="${build.dir}" defaultexcludes="yes">
            <include name="nativelibs/libvoltdb*" />
        </fileset>
    </copy>
    <copy todir="${dist.dir}/voltdb" flatten="true" >
        <fileset dir="." defaultexcludes="yes">
            <include name="third_party/cpp/jnilib/libjzmq*" />
        </fileset>
    </copy>

    <!-- save the symbols from the shared library -->
    <exec dir='${dist.dir}/voltdb' executable='/bin/sh'>
        <arg line="-c '/usr/bin/objcopy --only-keep-debug libvoltdb-${dist.version}.so ${build.dir}/voltdb-${dist.version}.sym'" />
    </exec>

    <!-- strip the voltbin shared library (~40x size reduction) -->
    <exec dir='${dist.dir}/voltdb' executable='/bin/sh'>
        <arg line="-c '/usr/bin/strip --strip-debug --strip-unneeded libvoltdb*'"/>
    </exec>

    <!-- embed the path to the symbol file into the ELF binary -->
    <exec dir='${dist.dir}/voltdb' executable='/bin/sh'>
        <arg line="-c '/usr/bin/objcopy --add-gnu-debuglink=voltdb-${dist.version}.sym libvoltdb-${dist.version}.so'" />
    </exec>

    <!-- copy studio.web -->
    <mkdir dir="${dist.dir}/tools/studio.web" />
    <copy todir="${dist.dir}/tools/studio.web">
        <fileset dir="${src.gpl.dir}/org/voltdb/studio" defaultexcludes="yes">
            <include name="**"/>
        </fileset>
    </copy>

    <!-- populate project generator -->
    <exec dir="src/proj_gen/" executable="python">
        <arg line="generator_compiler.py"/>
    </exec>
    <move tofile="${dist.dir}/tools/generate" file="src/proj_gen/generate" />

    <!-- add the Project and Deployment file schemas to the dist -->
    <copy todir="${dist.dir}/tools" file="src/frontend/org/voltdb/compiler/ProjectFileSchema.xsd"/>
    <copy todir="${dist.dir}/tools" file="src/frontend/org/voltdb/compiler/DeploymentFileSchema.xsd"/>

    <!-- copy license to voltdb dir -->
    <copy todir="${dist.dir}/voltdb" file="LICENSE"/>

    <!-- make shell scripts executable -->
    <chmod perm="ugo+rx">
        <fileset dir="${dist.dir}" defaultexcludes="yes">
            <include name="bin/*"/>
            <include name="tools/demo/launch"/>
            <include name="doc/tutorials/**/run.sh"/>
            <include name="examples/**/run.sh"/>
            <include name="tools/generate"/>
            <include name="tools/sqlcmd/sqlcmd"/>
        </fileset>
    </chmod>

    <!-- create an archive for distribution -->
    <exec executable="mv" failonerror="true">
        <arg value="${raw.dist.dir}/dist"/>
        <arg value="${raw.dist.dir}/voltdb-${dist.version}"/>
    </exec>
    <exec executable="tar" failonerror="true">
        <arg value="-cz"/>
        <arg value="-C"/>
        <arg value="${raw.dist.dir}"/>
        <arg value="-f"/>
        <arg value="${raw.dist.dir}/voltdb-${dist.version}.tar.gz"/>
        <arg value="voltdb-${dist.version}"/>
    </exec>
    <!-- move it back to dist directory for downstream dependencies -->
    <exec executable="mv" failonerror="true">
        <arg value="${raw.dist.dir}/voltdb-${dist.version}"/>
        <arg value="${raw.dist.dir}/dist"/>
    </exec>

    <!-- package up studio web trivially -->
    <zip destfile="${raw.dist.dir}/voltdb-studio.web-${dist.version}.zip">
        <fileset dir="${src.studio.dir}" defaultexcludes="yes">
            <include name="**/*" />
        </fileset>
    </zip>
</target>

<target name="dist_client" depends="dist_internal"
        description="Hacky Java client package target">
    <!-- prepare release directory for new content -->
    <delete includeemptydirs="true" failonerror='false'>
        <fileset dir="${dist.dir}-client-java" includes="**/*" />
    </delete>
    <mkdir dir="${dist.dir}-client-java" />

    <!-- populate the dist-client-java dir from the superset dist -->
    <copy todir="${dist.dir}-client-java/">
        <fileset dir="${dist.dir}/" defaultexcludes="yes">
            <include name="**/*"/>
            <exclude name=".gitignore"/>
            <exclude name="bin/"/>
            <exclude name="voltdb/libvoltdb*"/>
            <exclude name="voltdb/voltdb-*.jar"/>
            <exclude name="voltdb/libjzmq*"/>
        </fileset>
    </copy>

    <!-- make shell scripts executable -->
    <chmod perm="ugo+rx">
        <fileset dir="${dist.dir}-client-java/" defaultexcludes="yes">
            <include name="tools/demo/launch"/>
            <include name="doc/tutorials/auction/run.sh"/>
            <include name="doc/tutorials/helloworld/run.sh"/>
            <include name="examples/voltcache/run.sh"/>
            <include name="examples/voltkv/run.sh"/>
            <include name="examples/voter/run.sh"/>
            <include name="tools/generate"/>
            <include name="tools/sqlcmd/sqlcmd"/>
        </fileset>
    </chmod>

    <!-- create an archive for distribution -->
    <exec executable="mv" failonerror="true">
        <arg value="${raw.dist.dir}/dist-client-java"/>
        <arg value="${raw.dist.dir}/voltdb-client-java-${dist.version}"/>
    </exec>
    <exec executable="tar" failonerror="true">
        <arg value="-cz"/>
        <arg value="-C"/>
        <arg value="${raw.dist.dir}"/>
        <arg value="-f"/>
        <arg value="${raw.dist.dir}/voltdb-client-java-${dist.version}.tar.gz"/>
        <arg value="voltdb-client-java-${dist.version}"/>
    </exec>
    <!-- move it back to dist directory for downstream dependencies -->
    <exec executable="mv" failonerror="true">
        <arg value="${raw.dist.dir}/voltdb-client-java-${dist.version}"/>
        <arg value="${raw.dist.dir}/dist-client-java"/>
    </exec>
</target>

<macrodef name="refapp_macro"
        description="Build a kit that only includes a single reference app">
  <attribute name="appname"/>
  <sequential>

    <!-- prepare release directory for new content -->
    <delete includeemptydirs="true" failonerror='false'>
        <fileset dir="${dist.dir}-@{appname}" includes="**/*" />
    </delete>
    <mkdir dir="${dist.dir}-@{appname}" />
    <mkdir dir="${dist.dir}-@{appname}/@{appname}" />

    <!-- populate the examples -->
    <copy todir="${dist.dir}-@{appname}/@{appname}">
        <fileset dir="${dist.dir}/examples/@{appname}" defaultexcludes="yes">
            <include name="**/*"/>
            <exclude name=".gitignore"/>
        </fileset>
    </copy>

    <!-- populate the bin/lib/voltdb/etc... dirs -->
    <copy todir="${dist.dir}-@{appname}" >
        <fileset dir="${dist.dir}" defaultexcludes="yes">
            <include name="README" />
            <include name="LICENSE" />
            <include name="README.thirdparty" />
            <include name="voltdb/**" />
            <include name="lib/**" />
            <include name="bin/**" />
        </fileset>
    </copy>

    <!-- rewrite the run.sh script's relative directories -->
    <replace
        file="${dist.dir}-@{appname}/@{appname}/run.sh"
        token="../../"
        value="../"/>

    <!-- make shell scripts executable -->
    <chmod perm="ugo+rx">
        <fileset dir="${dist.dir}-@{appname}" defaultexcludes="yes">
            <include name="bin/*"/>
            <include name="@{appname}/run.sh"/>
        </fileset>
    </chmod>



    <!-- create an archive for distribution -->
    <exec executable="mv" failonerror="true">
        <arg value="${raw.dist.dir}/dist-@{appname}"/>
        <arg value="${raw.dist.dir}/voltdb-@{appname}-${dist.version}"/>
    </exec>
    <exec executable="tar" failonerror="true">
        <arg value="-cz"/>
        <arg value="-C"/>
        <arg value="${raw.dist.dir}"/>
        <arg value="-f"/>
        <arg value="${raw.dist.dir}/voltdb-@{appname}-${dist.version}.tar.gz"/>
        <arg value="voltdb-@{appname}-${dist.version}"/>
    </exec>
    <!-- move it back to dist directory for downstream dependencies -->
    <exec executable="mv" failonerror="true">
        <arg value="${raw.dist.dir}/voltdb-@{appname}-${dist.version}"/>
        <arg value="${raw.dist.dir}/dist-@{appname}"/>
    </exec>
  </sequential>
</macrodef>

<!--
***************************************
CLEANING
***************************************
-->

<target name='clean' description="Remove all compiled files." depends="cleantmp">
  <exec dir='examples/voter' executable='/usr/bin/env'><arg line="bash run.sh clean"/></exec>
  <exec dir='examples/voltcache' executable='/usr/bin/env'><arg line="bash run.sh clean"/></exec>
  <exec dir='examples/voltkv' executable='/usr/bin/env'><arg line="bash run.sh clean"/></exec>
  <exec dir='doc/tutorials/auction' executable='/usr/bin/env'><arg line="bash run.sh clean"/></exec>
  <exec dir='doc/tutorials/helloworld' executable='/usr/bin/env'><arg line="bash run.sh clean"/></exec>  <exec dir='.' executable='/bin/sh'>
    <arg line="-c 'rm -rf obj/*'"/>
  </exec>
  <exec dir='.' executable='/bin/sh'>
    <arg line="-c 'rm -rf voltdb/*.jar'"/>
  </exec>
  <exec dir='.' executable='/bin/sh'>
    <arg line="-c 'rm -rf voltdb/*.so voltdb/*.jnilib'"/>
  </exec>
  <exec dir='.' executable='/bin/sh'>
    <arg line="-c 'rm -rf src/ee/catalog/*'"/>
  </exec>
  <exec dir='.' executable='/bin/sh'>
    <arg line="-c 'rm -rf src/frontend/org/voltdb/catalog/*.java'"/>
{'fraction_non_alphanumeric': 0.1987675909734366, 'fraction_numerical': 0.0037888250478807562, 'mean_word_length': 2.8549875611909155, 'pattern_counts': {'":': 0, '<': 1005, '<?xml version=': 2, '>': 1005, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3537341', 'n_tokens_mistral': 15251, 'n_tokens_neox': 14737, 'n_words': 2772}