<commit_before><?xml version="1.0"?>

<project name="core" default="default" basedir=".">

  <property name="junit.includes" value="**/Test*.java"/>

  <import file="common-build.xml"/>

  <!-- Import tests for deprecation compliance.
       This will be removed for Lucene 2.0
   -->
  <import file="build-deprecated.xml"/>

  <property name="demo.name" value="lucene-demos-${version}"/>
  <property name="demo.war.name" value="luceneweb"/>
  <property name="javadoc.link" value="http://java.sun.com/j2se/1.4/docs/api/"/>

  <!-- Build classpath -->
  <path id="classpath">
    <pathelement location="${build.dir}/classes/java"/>
  </path>

  <path id="demo.classpath">
    <path refid="classpath"/>
    <pathelement location="${build.dir}/classes/demo"/>
  </path>

  <path id="test.classpath">
    <path refid="demo.classpath"/>
    <pathelement location="${build.dir}/classes/test"/>
  </path>


  <path id="junit.classpath">
    <pathelement location="${build.dir}/classes/test"/>
    <pathelement location="${build.dir}/classes/java"/>
    <pathelement location="${build.dir}/classes/demo"/>
    <fileset dir="lib">
      <include name="*.jar"/>
    </fileset>
    <pathelement path="${java.class.path}"/>
  </path>

  <patternset id="src.dist.patterns"
              includes="src/,build.xml,*build*.xml,docs/,*.txt,contrib/,"
              excludes="contrib/*/lib/"
  />
  <patternset id="binary.build.dist.patterns"
              includes="${final.name}.jar,${demo.war.name}.war,${demo.name}.jar,docs/,contrib/*/*.jar"
  />
  <patternset id="binary.root.dist.patterns"
              includes="src/demo/,src/jsp/,build.xml,docs/,*.txt"
  />


  <!-- ================================================================== -->
  <!-- Prepares the build directory                                       -->
  <!-- ================================================================== -->

  <!-- Overrides common.compile-core to add rmic -->
  <target name="compile-core" depends="common.compile-core"
          description="Compiles core classes, including rmic">
    <rmic classname="org.apache.lucene.search.RemoteSearchable"
      base="${build.dir}/classes/java">
      <classpath refid="classpath"/>
    </rmic>
  </target>

  <!-- ================================================================== -->
  <!-- J A R                                                              -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->

  <target name="jar-demo" depends="compile-demo">
    <jar
      destfile="${build.dir}/${demo.name}.jar"
      basedir="${build.dir}/classes/demo"
      excludes="**/*.java"
      />
  </target>

  <target name="war-demo" depends="jar-core,jar-demo">
    <war destfile="${build.dir}/${demo.war.name}.war"
         webxml="src/jsp/WEB-INF/web.xml">
      <fileset dir="src/jsp" excludes="WEB-INF/web.xml"/>
      <lib dir="${build.dir}" includes="${demo.name}.jar"/>
      <lib dir="${build.dir}" includes="${final.name}.jar"/>
    </war>
  </target>

  <!-- ================================================================== -->
  <!-- B U I L D  D E M O                                                 -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="compile-demo" depends="compile-core">
    <mkdir dir="${build.dir}/classes/demo"/>

    <compile
      srcdir="src/demo"
      destdir="${build.dir}/classes/demo">
      <classpath refid="demo.classpath"/>
    </compile>
  </target>

  <!-- GCJ -->
  <target name="gcj" depends="jar,jar-demo">
    <exec executable="make" dir="src/gcj" failonerror="true"/>
  </target>


  <!-- TODO: Remove dependency on demo from test -->
	<target name="compile-test" depends="compile-demo,common.compile-test"/>



  <!-- ================================================================== -->
  <!-- D O C U M E N T A T I O N                                          -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="docs">
    <ant dir="site" inheritAll="false"/>
  </target>

  <target name="javadocs">
    <mkdir dir="${build.dir}/docs/api"/>
    <invoke-javadoc
      access="protected"
      destdir="${build.dir}/docs/api"
    />
  </target>

  <target name="javadocs-internal">
    <mkdir dir="${build.dir}/docs/api-internal"/>
    <invoke-javadoc
      access="package"
      destdir="${build.dir}/docs/api-internal"
    />
  </target>


  <!-- ================================================================== -->
  <!-- D I S T R I B U T I O N                                            -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="package" depends="jar-core, javadocs, jjdoc, war-demo, build-contrib">
    <mkdir dir="${dist.dir}"/>
  </target>

  <!-- ================================================================== -->
  <!-- Packages the distribution with zip                                 -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="package-zip" depends="package"
    description="--> Generates the Lucene distribution as .zip">

    <delete file="${dist.dir}/lucene-${version}.zip"/>
    <zip destfile="${dist.dir}/lucene-${version}.zip">
      <zipfileset prefix="lucene-${version}" dir=".">
        <patternset refid="binary.root.dist.patterns"/>
      </zipfileset>
      <zipfileset prefix="lucene-${version}" dir="${build.dir}">
        <patternset refid="binary.build.dist.patterns"/>
      </zipfileset>
    </zip>
  </target>

  <!-- ================================================================== -->
  <!-- packages the distribution with tar-gzip                            -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="package-tgz" depends="package"
    description="--> Generates the lucene distribution as .tar.gz">

    <delete file="${build.dir}/lucene-${version}.tar"/>
    <delete file="${dist.dir}/lucene-${version}.tar.gz"/>
    <tar tarfile="${build.dir}/lucene-${version}.tar">
      <tarfileset prefix="lucene-${version}" dir=".">
        <patternset refid="binary.root.dist.patterns"/>
      </tarfileset>
      <tarfileset prefix="lucene-${version}" dir="${build.dir}">
        <patternset refid="binary.build.dist.patterns"/>
      </tarfileset>
    </tar>

    <gzip zipfile="${dist.dir}/lucene-${version}.tar.gz"
      src="${build.dir}/lucene-${version}.tar"
      />
  </target>

  <!-- ================================================================== -->
  <!-- packages the distribution with zip and tar-gzip                    -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="package-all-binary" depends="package-zip, package-tgz"
    description="--> Generates the .tar.gz and .zip distributions"/>

  <!-- ================================================================== -->
  <!-- same as package-all. it is just here for compatibility.            -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="dist" depends="package-all-binary"/>

  <!-- ================================================================== -->
  <!-- S O U R C E  D I S T R I B U T I O N                               -->
  <!-- ================================================================== -->

  <!-- ================================================================== -->
  <!-- Packages the sources with zip                                      -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="package-zip-src" depends="init"
    description="--> Generates the Lucene sources as .zip">
    <zip destfile="${dist.dir}/lucene-${version}-src.zip">
      <zipfileset prefix="lucene-${version}" dir=".">
        <patternset refid="src.dist.patterns"/>
      </zipfileset>
    </zip>
  </target>

  <!-- ================================================================== -->
  <!-- Packages the sources with tar-gzip                                 -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="package-tgz-src" depends="init"
    description="--> Generates the Lucene distribution as .tar.gz">

    <delete file="${build.dir}/lucene-${version}-src.tar"/>
    <delete file="${dist.dir}/lucene-${version}-src.tar.gz"/>
    <tar tarfile="${build.dir}/lucene-${version}-src.tar">
      <tarfileset prefix="lucene-${version}" dir=".">
        <patternset refid="src.dist.patterns"/>
      </tarfileset>
    </tar>

    <gzip zipfile="${dist.dir}/lucene-${version}-src.tar.gz"
          src="${build.dir}/lucene-${version}-src.tar"
    />
  </target>

  <!-- ================================================================== -->
  <!-- Packages the sources with zip and tar-gzip                         -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="package-all-src" depends="package-zip-src, package-tgz-src"
    description="--> Generates the .tar.gz and .zip source distributions"/>

  <!-- ================================================================== -->
  <!-- same as package-all-src. it is just here for compatibility.        -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="dist-src" depends="package-all-src"/>

  <target name="dist-all" depends="dist, dist-src"/>

  <!-- ================================================================== -->
  <!-- Build the JavaCC files into the source tree                        -->
  <!-- ================================================================== -->
  <target name="jjdoc">
    <mkdir dir="${build.dir}/docs/grammars"/>
    <jjdoc target="src/java/org/apache/lucene/queryParser/QueryParser.jj"
           outputfile="${build.dir}/docs/grammars/QueryParser.html"
           javacchome="${javacc.home}"
    />
    <jjdoc target="src/java/org/apache/lucene/analysis/standard/StandardTokenizer.jj"
           outputfile="${build.dir}/docs/grammars/StandardTokenizer.html"
           javacchome="${javacc.home}"
    />
    <jjdoc target="src/demo/org/apache/lucene/demo/html/HTMLParser.jj"
           outputfile="${build.dir}/docs/grammars/HTMLParser.html"
           javacchome="${javacc.home}"
    />
  </target>

  <target name="clean-javacc">
    <delete>
      <fileset dir="src">
        <include name="java/org/apache/lucene/analysis/standard/StandardTokenizer.java"/>
        <include name="java/org/apache/lucene/queryParser/QueryParser.java"/>
        <include name="demo/org/apache/lucene/demo/html/HTMLParser.java"/>
      </fileset>
    </delete>
  </target>

  <target name="javacc" depends="clean-javacc,javacc-StandardAnalyzer,javacc-QueryParser,javacc-HTMLParser"/>

  <target name="javacc-StandardAnalyzer" depends="init,javacc-check" if="javacc.present">
    <!-- generate this in a build directory so we can exclude ParseException -->
    <mkdir dir="${build.dir}/gen/org/apache/lucene/analysis/standard"/>

    <invoke-javacc target="src/java/org/apache/lucene/analysis/standard/StandardTokenizer.jj"
                   outputDir="${build.dir}/gen/org/apache/lucene/analysis/standard"
    />
    <copy todir="src/java/org/apache/lucene/analysis/standard">
      <fileset dir="${build.dir}/gen/org/apache/lucene/analysis/standard">
        <include name="*.java"/>
        <exclude name="ParseException.java"/>
      </fileset>
    </copy>
  </target>

  <target name="javacc-QueryParser" depends="init,javacc-check" if="javacc.present">
    <invoke-javacc target="src/java/org/apache/lucene/queryParser/QueryParser.jj"
                   outputDir="src/java/org/apache/lucene/queryParser"
    />
  </target>
  
  <target name="javacc-HTMLParser" depends="init,javacc-check" if="javacc.present">
    <invoke-javacc target="src/demo/org/apache/lucene/demo/html/HTMLParser.jj"
                   outputDir="src/demo/org/apache/lucene/demo/html"
    />
  </target>
  
  <macrodef name="contrib-crawl">
    <attribute name="target" default=""/>
    <sequential>
      <subant target="@{target}" failonerror="false">
        <fileset dir="."
                 includes="contrib/*/build.xml"
        />
      </subant>
    </sequential>
  </macrodef>

  <target name="build-contrib" depends="compile-test">
    <contrib-crawl/>
  </target>

  <target name="test-contrib" depends="compile-test">
    <contrib-crawl target="test"/>
  </target>

  <macrodef name="invoke-javadoc">
    <attribute name="access"/>
    <attribute name="destdir"/>
    <sequential>
      <javadoc
          overview="src/java/overview.html"
          packagenames="org.apache.lucene.*"
          destdir="@{destdir}"
          access="@{access}"
          encoding="${build.encoding}"
          author="true"
          version="true"
          use="true"
          link="${javadoc.link}"
          windowtitle="${Name} ${version} API"
          doctitle="${Name} ${version} API"
          bottom="Copyright &amp;copy; ${year} Apache Software Foundation.  All Rights Reserved.">
        <tag name="todo" description="To Do:"/>

        <!-- TODO: find a dynamic way to do include multiple source roots -->
        <packageset dir="src/java"/>
        <packageset dir="contrib/analyzers/src/java"/>
        <packageset dir="contrib/wordnet/src/java"/>
        <packageset dir="contrib/highlighter/src/java"/>
        <packageset dir="contrib/similarity/src/java"/>
        <packageset dir="contrib/spellchecker/src/java"/>
        <packageset dir="contrib/snowball/src/java" excludes="net/"/>
        <packageset dir="contrib/swing/src/java"/>
        <packageset dir="contrib/memory/src/java"/>

        <group title="Core" packages="org.apache.*:org.apache.lucene.analysis:org.apache.lucene.analysis.standard*"/>
        <group title="Analysis" packages="org.apache.lucene.analysis.*"/>
        <group title="Snowball" packages="org.apache.lucene.analysis.snowball*"/>
        <group title="Highlighter" packages="org.apache.lucene.search.highlight*"/>
        <group title="MoreLikeThis" packages="org.apache.lucene.search.similar*"/>
        <group title="SpellChecker" packages="org.apache.lucene.search.spell*"/>
        <group title="WordNet" packages="org.apache.lucene.wordnet*"/>
        <group title="Swing" packages="org.apache.lucene.swing*"/>
        <group title="Memory" packages="org.apache.lucene.index.memory*"/>
      </javadoc>
    </sequential>
  </macrodef>


</project>
<commit_msg>Make "ant -projecthelp" show the javadocs and docs targets as well (LUCENE-447), by <NAME><commit_after><?xml version="1.0"?>

<project name="core" default="default" basedir=".">

  <property name="junit.includes" value="**/Test*.java"/>

  <import file="common-build.xml"/>

  <!-- Import tests for deprecation compliance.
       This will be removed for Lucene 2.0
   -->
  <import file="build-deprecated.xml"/>

  <property name="demo.name" value="lucene-demos-${version}"/>
  <property name="demo.war.name" value="luceneweb"/>
  <property name="javadoc.link" value="http://java.sun.com/j2se/1.4/docs/api/"/>

  <!-- Build classpath -->
  <path id="classpath">
    <pathelement location="${build.dir}/classes/java"/>
  </path>

  <path id="demo.classpath">
    <path refid="classpath"/>
    <pathelement location="${build.dir}/classes/demo"/>
  </path>

  <path id="test.classpath">
    <path refid="demo.classpath"/>
    <pathelement location="${build.dir}/classes/test"/>
  </path>


  <path id="junit.classpath">
    <pathelement location="${build.dir}/classes/test"/>
    <pathelement location="${build.dir}/classes/java"/>
    <pathelement location="${build.dir}/classes/demo"/>
    <fileset dir="lib">
      <include name="*.jar"/>
    </fileset>
    <pathelement path="${java.class.path}"/>
  </path>

  <patternset id="src.dist.patterns"
              includes="src/,build.xml,*build*.xml,docs/,*.txt,contrib/,"
              excludes="contrib/*/lib/"
  />
  <patternset id="binary.build.dist.patterns"
              includes="${final.name}.jar,${demo.war.name}.war,${demo.name}.jar,docs/,contrib/*/*.jar"
  />
  <patternset id="binary.root.dist.patterns"
              includes="src/demo/,src/jsp/,build.xml,docs/,*.txt"
  />


  <!-- ================================================================== -->
  <!-- Prepares the build directory                                       -->
  <!-- ================================================================== -->

  <!-- Overrides common.compile-core to add rmic -->
  <target name="compile-core" depends="common.compile-core"
          description="Compiles core classes, including rmic">
    <rmic classname="org.apache.lucene.search.RemoteSearchable"
      base="${build.dir}/classes/java">
      <classpath refid="classpath"/>
    </rmic>
  </target>

  <!-- ================================================================== -->
  <!-- J A R                                                              -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->

  <target name="jar-demo" depends="compile-demo">
    <jar
      destfile="${build.dir}/${demo.name}.jar"
      basedir="${build.dir}/classes/demo"
      excludes="**/*.java"
      />
  </target>

  <target name="war-demo" depends="jar-core,jar-demo">
    <war destfile="${build.dir}/${demo.war.name}.war"
         webxml="src/jsp/WEB-INF/web.xml">
      <fileset dir="src/jsp" excludes="WEB-INF/web.xml"/>
      <lib dir="${build.dir}" includes="${demo.name}.jar"/>
      <lib dir="${build.dir}" includes="${final.name}.jar"/>
    </war>
  </target>

  <!-- ================================================================== -->
  <!-- B U I L D  D E M O                                                 -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="compile-demo" depends="compile-core">
    <mkdir dir="${build.dir}/classes/demo"/>

    <compile
      srcdir="src/demo"
      destdir="${build.dir}/classes/demo">
      <classpath refid="demo.classpath"/>
    </compile>
  </target>

  <!-- GCJ -->
  <target name="gcj" depends="jar,jar-demo">
    <exec executable="make" dir="src/gcj" failonerror="true"/>
  </target>


  <!-- TODO: Remove dependency on demo from test -->
	<target name="compile-test" depends="compile-demo,common.compile-test"/>



  <!-- ================================================================== -->
  <!-- D O C U M E N T A T I O N                                          -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="docs" description="Build the website">
    <ant dir="site" inheritAll="false"/>
  </target>

  <target name="javadocs" description="Generate javadoc">
    <mkdir dir="${build.dir}/docs/api"/>
    <invoke-javadoc
      access="protected"
      destdir="${build.dir}/docs/api"
    />
  </target>

  <target name="javadocs-internal">
    <mkdir dir="${build.dir}/docs/api-internal"/>
    <invoke-javadoc
      access="package"
      destdir="${build.dir}/docs/api-internal"
    />
  </target>


  <!-- ================================================================== -->
  <!-- D I S T R I B U T I O N                                            -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="package" depends="jar-core, javadocs, jjdoc, war-demo, build-contrib">
    <mkdir dir="${dist.dir}"/>
  </target>

  <!-- ================================================================== -->
  <!-- Packages the distribution with zip                                 -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="package-zip" depends="package"
    description="--> Generates the Lucene distribution as .zip">

    <delete file="${dist.dir}/lucene-${version}.zip"/>
    <zip destfile="${dist.dir}/lucene-${version}.zip">
      <zipfileset prefix="lucene-${version}" dir=".">
        <patternset refid="binary.root.dist.patterns"/>
      </zipfileset>
      <zipfileset prefix="lucene-${version}" dir="${build.dir}">
        <patternset refid="binary.build.dist.patterns"/>
      </zipfileset>
    </zip>
  </target>

  <!-- ================================================================== -->
  <!-- packages the distribution with tar-gzip                            -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="package-tgz" depends="package"
    description="--> Generates the lucene distribution as .tar.gz">

    <delete file="${build.dir}/lucene-${version}.tar"/>
    <delete file="${dist.dir}/lucene-${version}.tar.gz"/>
    <tar tarfile="${build.dir}/lucene-${version}.tar">
      <tarfileset prefix="lucene-${version}" dir=".">
        <patternset refid="binary.root.dist.patterns"/>
      </tarfileset>
      <tarfileset prefix="lucene-${version}" dir="${build.dir}">
        <patternset refid="binary.build.dist.patterns"/>
      </tarfileset>
    </tar>

    <gzip zipfile="${dist.dir}/lucene-${version}.tar.gz"
      src="${build.dir}/lucene-${version}.tar"
      />
  </target>

  <!-- ================================================================== -->
  <!-- packages the distribution with zip and tar-gzip                    -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="package-all-binary" depends="package-zip, package-tgz"
    description="--> Generates the .tar.gz and .zip distributions"/>

  <!-- ================================================================== -->
  <!-- same as package-all. it is just here for compatibility.            -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="dist" depends="package-all-binary"/>

  <!-- ================================================================== -->
  <!-- S O U R C E  D I S T R I B U T I O N                               -->
  <!-- ================================================================== -->

  <!-- ================================================================== -->
  <!-- Packages the sources with zip                                      -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="package-zip-src" depends="init"
    description="--> Generates the Lucene sources as .zip">
    <zip destfile="${dist.dir}/lucene-${version}-src.zip">
      <zipfileset prefix="lucene-${version}" dir=".">
        <patternset refid="src.dist.patterns"/>
      </zipfileset>
    </zip>
  </target>

  <!-- ================================================================== -->
  <!-- Packages the sources with tar-gzip                                 -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="package-tgz-src" depends="init"
    description="--> Generates the Lucene distribution as .tar.gz">

    <delete file="${build.dir}/lucene-${version}-src.tar"/>
    <delete file="${dist.dir}/lucene-${version}-src.tar.gz"/>
    <tar tarfile="${build.dir}/lucene-${version}-src.tar">
      <tarfileset prefix="lucene-${version}" dir=".">
        <patternset refid="src.dist.patterns"/>
      </tarfileset>
    </tar>

    <gzip zipfile="${dist.dir}/lucene-${version}-src.tar.gz"
          src="${build.dir}/lucene-${version}-src.tar"
    />
  </target>

  <!-- ================================================================== -->
  <!-- Packages the sources with zip and tar-gzip                         -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="package-all-src" depends="package-zip-src, package-tgz-src"
    description="--> Generates the .tar.gz and .zip source distributions"/>

  <!-- ================================================================== -->
  <!-- same as package-all-src. it is just here for compatibility.        -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="dist-src" depends="package-all-src"/>

  <target name="dist-all" depends="dist, dist-src"/>

  <!-- ================================================================== -->
  <!-- Build the JavaCC files into the source tree                        -->
  <!-- ================================================================== -->
  <target name="jjdoc">
    <mkdir dir="${build.dir}/docs/grammars"/>
    <jjdoc target="src/java/org/apache/lucene/queryParser/QueryParser.jj"
           outputfile="${build.dir}/docs/grammars/QueryParser.html"
           javacchome="${javacc.home}"
    />
    <jjdoc target="src/java/org/apache/lucene/analysis/standard/StandardTokenizer.jj"
           outputfile="${build.dir}/docs/grammars/StandardTokenizer.html"
           javacchome="${javacc.home}"
    />
    <jjdoc target="src/demo/org/apache/lucene/demo/html/HTMLParser.jj"
           outputfile="${build.dir}/docs/grammars/HTMLParser.html"
           javacchome="${javacc.home}"
    />
  </target>

  <target name="clean-javacc">
    <delete>
      <fileset dir="src">
        <include name="java/org/apache/lucene/analysis/standard/StandardTokenizer.java"/>
        <include name="java/org/apache/lucene/queryParser/QueryParser.java"/>
        <include name="demo/org/apache/lucene/demo/html/HTMLParser.java"/>
      </fileset>
    </delete>
  </target>

  <target name="javacc" depends="clean-javacc,javacc-StandardAnalyzer,javacc-QueryParser,javacc-HTMLParser"/>

  <target name="javacc-StandardAnalyzer" depends="init,javacc-check" if="javacc.present">
    <!-- generate this in a build directory so we can exclude ParseException -->
    <mkdir dir="${build.dir}/gen/org/apache/lucene/analysis/standard"/>

    <invoke-javacc target="src/java/org/apache/lucene/analysis/standard/StandardTokenizer.jj"
                   outputDir="${build.dir}/gen/org/apache/lucene/analysis/standard"
    />
    <copy todir="src/java/org/apache/lucene/analysis/standard">
      <fileset dir="${build.dir}/gen/org/apache/lucene/analysis/standard">
        <include name="*.java"/>
        <exclude name="ParseException.java"/>
      </fileset>
    </copy>
  </target>

  <target name="javacc-QueryParser" depends="init,javacc-check" if="javacc.present">
    <invoke-javacc target="src/java/org/apache/lucene/queryParser/QueryParser.jj"
                   outputDir="src/java/org/apache/lucene/queryParser"
    />
  </target>
  
  <target name="javacc-HTMLParser" depends="init,javacc-check" if="javacc.present">
    <invoke-javacc target="src/demo/org/apache/lucene/demo/html/HTMLParser.jj"
                   outputDir="src/demo/org/apache/lucene/demo/html"
    />
  </target>
  
  <macrodef name="contrib-crawl">
    <attribute name="target" default=""/>
    <sequential>
      <subant target="@{target}" failonerror="false">
        <fileset dir="."
                 includes="contrib/*/build.xml"
        />
      </subant>
    </sequential>
  </macrodef>

  <target name="build-contrib" depends="compile-test">
    <contrib-crawl/>
  </target>

  <target name="test-contrib" depends="compile-test">
    <contrib-crawl target="test"/>
  </target>

  <macrodef name="invoke-javadoc">
    <attribute name="access"/>
    <attribute name="destdir"/>
    <sequential>
      <javadoc
          overview="src/java/overview.html"
          packagenames="org.apache.lucene.*"
          destdir="@{destdir}"
          access="@{access}"
          encoding="${build.encoding}"
          author="true"
          version="true"
          use="true"
          link="${javadoc.link}"
          windowtitle="${Name} ${version} API"
          doctitle="${Name} ${version} API"
          bottom="Copyright &amp;copy; ${year} Apache Software Foundation.  All Rights Reserved.">
        <tag name="todo" description="To Do:"/>

        <!-- TODO: find a dynamic way to do include multiple source roots -->
        <packageset dir="src/java"/>
        <packageset dir="contrib/analyzers/src/java"/>
        <packageset dir="contrib/wordnet/src/java"/>
        <packageset dir="contrib/highlighter/src/java"/>
        <packageset dir="contrib/similarity/src/java"/>
        <packageset dir="contrib/spellchecker/src/java"/>
        <packageset dir="contrib/snowball/src/java" excludes="net/"/>
        <packageset dir="contrib/swing/src/java"/>
        <packageset dir="contrib/memory/src/java"/>

        <group title="Core" packages="org.apache.*:org.apache.lucene.analysis:org.apache.lucene.analysis.standard*"/>
        <group title="Analysis" packages="org.apache.lucene.analysis.*"/>
        <group title="Snowball" packages="org.apache.lucene.analysis.snowball*"/>
        <group title="Highlighter" packages="org.apache.lucene.search.highlight*"/>
        <group title="MoreLikeThis" packages="org.apache.lucene.search.similar*"/>
        <group title="SpellChecker" packages="org.apache.lucene.search.spell*"/>
        <group title="WordNet" packages="org.apache.lucene.wordnet*"/>
        <group title="Swing" packages="org.apache.lucene.swing*"/>
        <group title="Memory" packages="org.apache.lucene.index.memory*"/>
      </javadoc>
    </sequential>
  </macrodef>


</project>

{'fraction_non_alphanumeric': 0.32800461081113874, 'fraction_numerical': 0.0005156828247285082, 'mean_word_length': 2.5353351206434316, 'pattern_counts': {'":': 0, '<': 536, '<?xml version=': 2, '>': 548, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 84, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7162356', 'n_tokens_mistral': 8705, 'n_tokens_neox': 8064, 'n_words': 1324}