<commit_before><project name="Generate various scala packages for parser needs" default="rebuild.all">
  <property file="scala.properties"/>
  <property name="base.dir" value="${project.lib}/../../.."/>


  <path id="base.path">
    <fileset dir="${scala.home}/lib">
      <include name="**/*.jar"/>
    </fileset>
    <fileset dir="${idea.lib}">
      <include name="**/*.jar"/>
    </fileset>
    <fileset dir="${ant.lib}">
      <include name="**/*.jar"/>
    </fileset>
    <fileset dir="${project.lib}">
      <include name="**/*.class"/>
    </fileset>
  </path>

  <path id="test.path">
    <pathelement location="${scala.home}\lib\scala-library.jar"/>
    <pathelement location="${idea.lib}\junit-4.0.jar"/>
  </path>

  <target name="clear.all" description="Clears results">
    <mkdir dir="${project.lib}"/>
    <delete includeemptydirs="true">
      <fileset dir="${project.lib}"/>
    </delete>
  </target>

  <target name="create.lexer" description="Makes lexer file">
    <ant dir="org/jetbrains/plugins/scala/lang/lexer/"
         antfile="build.xml" target="generate.lexer"/>
  </target>

  <target name="make.project" depends="create.lexer" description="Builds all classes">

    <mkdir dir="${project.lib}/../"/>
    <mkdir dir="${project.lib}"/>
    <mkdir dir="${project.lib}/META-INF"/>

    <copy todir="${project.lib}/META-INF">
      <fileset dir="../META-INF"/>
    </copy>

    <copy todir="${project.lib}">
      <fileset dir="../resources/"/>
    </copy>

    <javac srcdir="."
           destdir="../classes/production/Scala">
      <classpath refid="base.path"/>
    </javac>

    <!--Scala taskdef-->
    <taskdef name="scalac"
             classname="org.jetbrains.plugins.scala.util.AntScalaCompiler"
             classpath="${project.lib}">
      <classpath>
        <pathelement location="${scala.home}/lib/scala-compiler.jar"/>
        <pathelement location="${scala.home}/lib/scala-library.jar"/>
      </classpath>
    </taskdef>

    <scalac srcdir="."
            destdir="${project.lib}"
            force="changed">
      <classpath refid="base.path"/>
      <include name="**/*.scala"/>
    </scalac>
  </target>

  <target name="build.tests">
    <javac srcdir="../test"
           destdir="../classes/production/Scala">
      <classpath refid="base.path"/>
    </javac>
  </target>

  <target name="rebuild.all" depends="clear.all, make.project" description="Builds all classes"/>

  <target name="move.to.sandbox" depends="make.project">
    <mkdir dir="C:\Documents and Settings\Ilya.Sergey\.IntelliJIdea70\sandbox\plugins\Scala\classes"/>
    <mkdir dir="C:\Documents and Settings\Ilya.Sergey\.IntelliJIdea70\sandbox\plugins\Scala\lib"/>
    <copy todir="C:\Documents and Settings\Ilya.Sergey\.IntelliJIdea70\sandbox\plugins\Scala\classes">
      <fileset dir="${project.lib}"/>
    </copy>
    <copy todir="C:\Documents and Settings\Ilya.Sergey\.IntelliJIdea70\sandbox\plugins\Scala\lib">
      <fileset dir="${scala.home}/lib"/>
    </copy>
    <copy todir="C:\Documents and Settings\Ilya.Sergey\.IntelliJIdea70\sandbox\plugins\Scala\META-INF">
      <fileset dir="${project.lib}/Meta-Inf"/>
    </copy>
  </target>


  <target name="test.formatter" depends="build.tests">

    <junit fork="no"
           dir="${base.dir}"
           printsummary="yes"
           haltonfailure="no">

      <classpath>
        <path location="${project.lib}"/>
        <path location="${idea.lib}"/>
        <path location="${ant.lib}"/>
        <path location="${scala.home}/lib"/>
      </classpath>

      <jvmarg value="-Djava.awt.headless=true"/>
      <jvmarg value="-Didea.plugins.load=false"/>
      <jvmarg value="-ea"/>
      <jvmarg value="-Xmx256M"/>
      <jvmarg value="-Xbootclasspath/p:${idea.lib}/boot.jar"/>

<commit_msg>New test<commit_after><project name="Generate various scala packages for parser needs" default="rebuild.all">
  <property file="scala.properties"/>
  <property name="base.dir" value="${project.lib}/../../.."/>


  <path id="base.path">
    <fileset dir="${scala.home}/lib">
      <include name="**/*.jar"/>
    </fileset>
    <fileset dir="${idea.lib}">
      <include name="**/*.jar"/>
    </fileset>
    <fileset dir="${ant.lib}">
      <include name="**/*.jar"/>
    </fileset>
    <fileset dir="${project.lib}">
      <include name="**/*.class"/>
    </fileset>
  </path>

  <path id="test.path">
    <pathelement location="${scala.home}\lib\scala-library.jar"/>
    <pathelement location="${idea.lib}\junit-4.0.jar"/>
  </path>

  <target name="clear.all" description="Clears results">
    <mkdir dir="${project.lib}"/>
    <delete includeemptydirs="true">
      <fileset dir="${project.lib}"/>
    </delete>
  </target>

  <target name="create.lexer" description="Makes lexer file">
    <ant dir="org/jetbrains/plugins/scala/lang/lexer/"
         antfile="build.xml" target="generate.lexer"/>
  </target>

  <target name="make.project" depends="create.lexer" description="Builds all classes">

    <mkdir dir="${project.lib}/../"/>
    <mkdir dir="${project.lib}"/>
    <mkdir dir="${project.lib}/META-INF"/>

    <copy todir="${project.lib}/META-INF">
      <fileset dir="../META-INF"/>
    </copy>

    <copy todir="${project.lib}">
      <fileset dir="../resources/"/>
    </copy>

    <javac srcdir="."
           destdir="../classes/production/Scala">
      <classpath refid="base.path"/>
    </javac>

    <!--Scala taskdef-->
    <taskdef name="scalac"
             classname="org.jetbrains.plugins.scala.util.AntScalaCompiler"
             classpath="${project.lib}">
      <classpath>
        <pathelement location="${scala.home}/lib/scala-compiler.jar"/>
        <pathelement location="${scala.home}/lib/scala-library.jar"/>
      </classpath>
    </taskdef>

    <scalac srcdir="."
            destdir="${project.lib}"
            force="changed">
      <classpath refid="base.path"/>
      <include name="**/*.scala"/>
    </scalac>
  </target>

  <target name="build.tests">
    <javac srcdir="../test"
           destdir="../classes/production/Scala">
      <classpath refid="base.path"/>
    </javac>
  </target>

  <target name="rebuild.all" depends="clear.all, make.project" description="Builds all classes"/>

  <target name="move.to.sandbox" depends="make.project">
    <mkdir dir="C:\Documents and Settings\Ilya.Sergey\.IntelliJIdea70\sandbox\plugins\Scala\classes"/>
    <mkdir dir="C:\Documents and Settings\Ilya.Sergey\.IntelliJIdea70\sandbox\plugins\Scala\lib"/>
    <copy todir="C:\Documents and Settings\Ilya.Sergey\.IntelliJIdea70\sandbox\plugins\Scala\classes">
      <fileset dir="${project.lib}"/>
    </copy>
    <copy todir="C:\Documents and Settings\Ilya.Sergey\.IntelliJIdea70\sandbox\plugins\Scala\lib">
      <fileset dir="${scala.home}/lib"/>
    </copy>
    <copy todir="C:\Documents and Settings\Ilya.Sergey\.IntelliJIdea70\sandbox\plugins\Scala\META-INF">
      <fileset dir="${project.lib}/Meta-Inf"/>
    </copy>
  </target>


  <target name="test.formatter" depends="build.tests">

    <junit fork="no"
           dir="${base.dir}"
           printsummary="yes"
           haltonfailure="no">

      <classpath>
        <path location="${project.lib}"/>
        <path location="${idea.lib}"/>
        <path location="${idea.lib}/../jre/jre/lib"/>
        <path location="${ant.lib}"/>
        <path location="${scala.home}/lib"/>
      </classpath>

      <jvmarg value="-Djava.awt.headless=true"/>
      <jvmarg value="-Didea.plugins.load=false"/>
      <jvmarg value="-ea"/>
      <jvmarg value="-Xmx256M"/>
      <jvmarg value="-Xbootclasspath/p:${idea.lib}/boot.jar"/>


{'fraction_non_alphanumeric': 0.20824009477425298, 'fraction_numerical': 0.003948927208108464, 'mean_word_length': 2.912461380020597, 'pattern_counts': {'":': 0, '<': 178, '<?xml version=': 0, '>': 178, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1025145', 'n_tokens_mistral': 2633, 'n_tokens_neox': 2483, 'n_words': 391}