<commit_before><!--

  This is the project type for data sources.
  A data source contains model additions, mappings, and code to perform
  retrieving/translation/loading.

  Each data source project could be used multiple times for a single
  production build, e.g. loading different data with slightly different settings.
  So this project type is a little unusual in that most targets (with the
  exception of the default target which simple builds the java) require
  several properties to be defined when invoked (with with -D on the command
  line or passed with <property ...> via an ant 'ant' task).

  Automatically set properties:

  source.name  - the name of the current source

  Properties to pass in:

  intermine.properties.file           - the name of the file in the home directory
  default.intermine.properties.file   - relative path to default intermine properties
  path.to.production.model.jar        - core model java jar
                                        (not quire sure how to handle this yet)
  common.tgt.os.prefix                - used to construct other property names
                                        for objectstores e.g.
                                        os.${common.os.prefix}-src-items-std

  Properties likely to be defined in project.properties:

  source.model       - source model name
  target.model       - target model name
  convertor.class    - if retrieving from file, file convertor class
  translator.class   - data translator class
  organism           - organism passed to translator

-->

<project name="source" default="default" basedir="."
  xmlns:source="http://www.intermine.org/ns/im-source-proj/1"
  xmlns:task="http://www.intermine.org/ns/im-task-proj/1">

  <dirname property="source.xml.basedir" file="${ant.file.source}"/>

  <import file="${source.xml.basedir}/library.xml"/>
  <import file="${source.xml.basedir}/task.xml"/>

  <!--
    Add the production model jar to the task classpath
  -->
  <target name="-define-task-classpath">
    <path id="task.class.path">
      <pathelement location="${build.task.dir}"/>
      <path refid="project.execute.path"/>
      <!-- And add anything in this project -->
      <!-- FIXED just don't allow tasks in this project - mst be in subproject -->
      <!--<pathelement location="${dist.jar}"/>-->
    </path>
  </target>


  <target name="-init-deps" depends="-init-properties, library.-init-deps, -init-task-xml">
    <!-- Copy resources into classpath -->
    <copy todir="${build.task.dir}">
      <fileset dir="${resources.dir}"/>
    </copy>
  </target>

  <!--
    Flags that control the integration process:
      have.db.src - the DB is put into a src items objectstore, translation needed
      have.file.xml.src - source is XML that is read into a src items objectstore, translate needed
      have.file.gff - source is read from a GFF file into a tgt items objectstore, no translation
      have.file.custom.src - read from a custom file into a tgt items objectstore, translate needed
      have.file.custom.tgt - read from a custom file into a src items objectstore, no translation
      have.file.dag - this source in read from a DAG file into a tgt items objectstore, no translate
      have.file.xml.tgt - source is XML that is read into a tgt items objectstore, no translation
      have.file.custom - source is read from a custom file using a DirectDataLoader, no translation,
                         no src or tgt items


    For the future:
      have.db.tgt
        - the DB is put into a tgt items objectstore, no translate
        - eg. when the the source pulls things directly from a foreign DB into
          a tgt items DB without using a translator
    -->
  <target name="-init-flags">
    <condition property="do.translate">
      <or>
        <istrue value="${have.db.src}"/>
        <istrue value="${have.file.xml.src}"/>
        <istrue value="${have.file.custom.src}"/>
      </or>
    </condition>
    <condition property="do.build.src.db">
      <istrue value="${do.translate}"/>
    </condition>
    <condition property="do.src.retrieve">
      <istrue value="${do.translate}"/>
    </condition>
    <condition property="do.analyse.db.src.items">
      <istrue value="${do.translate}"/>
    </condition>
    <condition property="do.tgt.retrieve">
      <isfalse value="${do.translate}"/>
    </condition>

    <!-- only initialise the tgt items database if it's needed -->
    <condition property="do.build.tgt.db">
      <isfalse value="${have.file.custom.direct}"/>
    </condition>
    <condition property="do.analyse.db.tgt.items">
      <isfalse value="${have.file.custom.direct}"/>
    </condition>
  </target>

  <target name="default">
    <echo message="project doesn't need compilation"/>
  </target>

  <target name="build-db-src-items" depends="init, -init-deps, -init-flags" if="do.build.src.db">
    <task:build-db
      model="fulldata"
      osname="os.${common.os.prefix}-src-items-std"/>
  </target>

  <target name="build-db-tgt-items" depends="init, -init-deps, -init-flags" if="do.build.tgt.db">
    <task:build-db
      model="fulldata"
      osname="os.${common.os.prefix}-tgt-items-std"/>
  </target>

  <!--
    Do stuff before retrieve. This might involve filtering the source data
    (see flybase-gff and uniprot)
  -->
  <target name="-pre-retrieve" depends="-init-flags">
    <!-- build src model if required -->
    <!-- filter -->
    <!-- alternatively, just depend on a source model project (e.g. sources/ensembl/model) -->
  </target>

  <!--
    Do stuff before translate
  -->
  <target name="-pre-translate" depends="-init-flags"/>

  <!--
    Do stuff before load.
  -->
  <target name="-pre-load" depends="-init-flags">
    <property name="ignore.duplicates" value="false"/>
    <echo message="indexing: os.${common.os.prefix}-tgt-items-std"/>
    <create-indexes alias="os.${common.os.prefix}-tgt-items-std"/>
    <analyse-db osName="os.${common.os.prefix}-tgt-items-std"/>
  </target>

  <!--
    Retrieve from a file in a src items db with the convertor specified via the property
    convertor.class
  -->
  <target name="-retrieve-src-from-custom-file" if="have.file.custom.src">
    <convert-file clsName="${convertor.class}"
                  osName="osw.${common.os.prefix}-src-items"
                  model="${target.model}"
                  dynamicAttrs="${dynamicAttrs}">
      <fileset dir="${src.data.dir}"/>
    </convert-file>
  </target>
<commit_msg>don't re-create tgt-items indexes before load<commit_after><!--

  This is the project type for data sources.
  A data source contains model additions, mappings, and code to perform
  retrieving/translation/loading.

  Each data source project could be used multiple times for a single
  production build, e.g. loading different data with slightly different settings.
  So this project type is a little unusual in that most targets (with the
  exception of the default target which simple builds the java) require
  several properties to be defined when invoked (with with -D on the command
  line or passed with <property ...> via an ant 'ant' task).

  Automatically set properties:

  source.name  - the name of the current source

  Properties to pass in:

  intermine.properties.file           - the name of the file in the home directory
  default.intermine.properties.file   - relative path to default intermine properties
  path.to.production.model.jar        - core model java jar
                                        (not quire sure how to handle this yet)
  common.tgt.os.prefix                - used to construct other property names
                                        for objectstores e.g.
                                        os.${common.os.prefix}-src-items-std

  Properties likely to be defined in project.properties:

  source.model       - source model name
  target.model       - target model name
  convertor.class    - if retrieving from file, file convertor class
  translator.class   - data translator class
  organism           - organism passed to translator

-->

<project name="source" default="default" basedir="."
  xmlns:source="http://www.intermine.org/ns/im-source-proj/1"
  xmlns:task="http://www.intermine.org/ns/im-task-proj/1">

  <dirname property="source.xml.basedir" file="${ant.file.source}"/>

  <import file="${source.xml.basedir}/library.xml"/>
  <import file="${source.xml.basedir}/task.xml"/>

  <!--
    Add the production model jar to the task classpath
  -->
  <target name="-define-task-classpath">
    <path id="task.class.path">
      <pathelement location="${build.task.dir}"/>
      <path refid="project.execute.path"/>
      <!-- And add anything in this project -->
      <!-- FIXED just don't allow tasks in this project - mst be in subproject -->
      <!--<pathelement location="${dist.jar}"/>-->
    </path>
  </target>


  <target name="-init-deps" depends="-init-properties, library.-init-deps, -init-task-xml">
    <!-- Copy resources into classpath -->
    <copy todir="${build.task.dir}">
      <fileset dir="${resources.dir}"/>
    </copy>
  </target>

  <!--
    Flags that control the integration process:
      have.db.src - the DB is put into a src items objectstore, translation needed
      have.file.xml.src - source is XML that is read into a src items objectstore, translate needed
      have.file.gff - source is read from a GFF file into a tgt items objectstore, no translation
      have.file.custom.src - read from a custom file into a tgt items objectstore, translate needed
      have.file.custom.tgt - read from a custom file into a src items objectstore, no translation
      have.file.dag - this source in read from a DAG file into a tgt items objectstore, no translate
      have.file.xml.tgt - source is XML that is read into a tgt items objectstore, no translation
      have.file.custom - source is read from a custom file using a DirectDataLoader, no translation,
                         no src or tgt items


    For the future:
      have.db.tgt
        - the DB is put into a tgt items objectstore, no translate
        - eg. when the the source pulls things directly from a foreign DB into
          a tgt items DB without using a translator
    -->
  <target name="-init-flags">
    <condition property="do.translate">
      <or>
        <istrue value="${have.db.src}"/>
        <istrue value="${have.file.xml.src}"/>
        <istrue value="${have.file.custom.src}"/>
      </or>
    </condition>
    <condition property="do.build.src.db">
      <istrue value="${do.translate}"/>
    </condition>
    <condition property="do.src.retrieve">
      <istrue value="${do.translate}"/>
    </condition>
    <condition property="do.analyse.db.src.items">
      <istrue value="${do.translate}"/>
    </condition>
    <condition property="do.tgt.retrieve">
      <isfalse value="${do.translate}"/>
    </condition>

    <!-- only initialise the tgt items database if it's needed -->
    <condition property="do.build.tgt.db">
      <isfalse value="${have.file.custom.direct}"/>
    </condition>
    <condition property="do.analyse.db.tgt.items">
      <isfalse value="${have.file.custom.direct}"/>
    </condition>
  </target>

  <target name="default">
    <echo message="project doesn't need compilation"/>
  </target>

  <target name="build-db-src-items" depends="init, -init-deps, -init-flags" if="do.build.src.db">
    <task:build-db
      model="fulldata"
      osname="os.${common.os.prefix}-src-items-std"/>
  </target>

  <target name="build-db-tgt-items" depends="init, -init-deps, -init-flags" if="do.build.tgt.db">
    <task:build-db
      model="fulldata"
      osname="os.${common.os.prefix}-tgt-items-std"/>
  </target>

  <!--
    Do stuff before retrieve. This might involve filtering the source data
    (see flybase-gff and uniprot)
  -->
  <target name="-pre-retrieve" depends="-init-flags">
    <!-- build src model if required -->
    <!-- filter -->
    <!-- alternatively, just depend on a source model project (e.g. sources/ensembl/model) -->
  </target>

  <!--
    Do stuff before translate
  -->
  <target name="-pre-translate" depends="-init-flags"/>

  <!--
    Do stuff before load.
  -->
  <target name="-pre-load" depends="-init-flags">
    <property name="ignore.duplicates" value="false"/>
  </target>

  <!--
    Retrieve from a file in a src items db with the convertor specified via the property
    convertor.class
  -->
  <target name="-retrieve-src-from-custom-file" if="have.file.custom.src">
    <convert-file clsName="${convertor.class}"
                  osName="osw.${common.os.prefix}-src-items"
                  model="${target.model}"
                  dynamicAttrs="${dynamicAttrs}">
      <fileset dir="${src.data.dir}"/>
    </convert-file>
  </target>

{'fraction_non_alphanumeric': 0.13425708470330358, 'fraction_numerical': 0.0003131360576170346, 'mean_word_length': 2.6189801699716715, 'pattern_counts': {'":': 0, '<': 164, '<?xml version=': 0, '>': 164, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7085682', 'n_tokens_mistral': 3702, 'n_tokens_neox': 3617, 'n_words': 1142}