<commit_before><tool id="repeat_masker" name="RepeatMasker" version="0.1.3" profile="17.01">
  <description>RepeatMasker</description>

  <requirements>
    <requirement type="package">repeatmasker</requirement>
  </requirements>

  <command detect_errors="exit_code"><![CDATA[
    RM_LIB_PATH=\$(dirname \$(which RepeatMasker))/../share/RepeatMasker/Libraries &&
    mkdir lib &&
    export REPEATMASKER_LIB_DIR=\$(pwd)/lib &&
      for file in \$(ls \$RM_LIB_PATH) ; do  ln -s \$RM_LIB_PATH/\$file lib/\$file ; done &&
    #if $repeat_source.source_type == "repbase":
      cp '${repeat_source.repbase_file}' lib/RMRBSeqs.embl &&
    #end if
    RepeatMasker -dir \$(pwd)
    #if $repeat_source.source_type == "library":
      -lib '${repeat_source.repeat_lib}'
      -cutoff '${repeat_source.cutoff}'
    #else if $repeat_source.source_type == "repbase":
      -species '${repeat_source.species}'
    #end if
    -parallel \${GALAXY_SLOTS:-1}
    '${gff}'
    '${ignore_n_stretches}'
    #if $advanced.show == 'yes':
      '${advanced.is_only}'
      '${advanced.is_clip}'
      '${advanced.no_is}'
      '${advanced.rodspec}'
      '${advanced.primspec}'
      '${advanced.nolow}'
      '${advanced.noint}'
      '${advanced.norna}'
      '${advanced.alu}'
      '${advanced.div}'
      '${advanced.search_speed}'
      '${advanced.frag}'
      '${advanced.maxsize}'
      #if $advanced.gc != -1:
        '${advanced.gc}'
      #end if
      '${advanced.gccalc}'
      '${advanced.nocut}'
      '${advanced.keep_alignments}'
      '${advanced.invert_alignments}'
      '${advanced.xout}'
      '${advanced.xsmall}'
      '${advanced.poly}'
    #end if
    '${input_fasta}' &&
    ls &&
    #if $advanced.show == 'no' or ($advanced.show == 'yes' and $advanced.is_only != '-is_only'):
      mv \$(basename '${input_fasta}').masked '${output_masked_genome}' &&
      sed -r 's/^ *// ; s/ *$//; s/\+ //; s/ +/\t/g ;  1,2c SW score\t% div.\t% del.\t% ins.\tquery sequence\tpos in  query: begin\tend\t(left)\trepeat\tclass/family\tpos in repeat: begin\tend\t(left)\tID' \$(basename '${input_fasta}').out >'${output_log}' &&
      mv \$(basename '${input_fasta}').tbl '${output_table}' &&
      #if $gff == '-gff':
        mv \$(basename '${input_fasta}').out.gff '${output_gff}' &&
      #end if
      #if $advanced.show == 'yes' and $advanced.keep_alignments == '-ali':
        mv \$(basename '${input_fasta}').align '${output_alignment}' &&
      #end if
      #if $advanced.show == 'yes' and $advanced.poly == '-poly':
        sed -r 's/^ *// ; s/ *$//; s/\+ //; s/ +/\t/g' \$(basename '${input_fasta}').polyout >'${output_polymorphic}' &&
      #end if
    #end if
    mv \$(basename '${input_fasta}').cat '${output_repeat_catalog}'
    ]]>
  </command>

  <inputs>
    <param name="input_fasta" type="data" format="fasta" label="Genomic DNA" />
    <conditional name="repeat_source">
      <param label="Repeat library source" name="source_type" type="select">
        <option selected="true" value="repbase">RepBase</option>
        <option value="library">Custom library of repeats</option>
      </param>
      <when value="repbase">
        <param name="repbase_file" type="data" format="embl" label="RepBase (RMRBSeqs.embl) file" />
        <param name="species" type="text" value="homo sapiens" label="Repeat source species" help="Source species (or clade name) used to select repeats from RepBase" />
      </when>
      <when value="library">
        <param name="repeat_lib" type="data" format="fasta" label="Custom library of repeats" />
        <param name="cutoff" type="integer" argument="-cutoff" value="225" label="Cutoff score for masking repeats" />
      </when>
    </conditional>
    <param name="gff" type="boolean" argument="-gff" truevalue="-gff" falsevalue="" label="Output annotation of repeats in GFF format" checked="false" />
    <param name="ignore_n_stretches" type="boolean" argument="-excln" falsevalue="" label="Ignore stretches of Ns when computing statistics" checked="true" help="Scaffolds are sometimes joined with stretches of 25 or more Ns. This option ignores them when calculating repeat statistics" />
    <conditional name="advanced">
      <param name="show" type="boolean" truevalue="yes" falsevalue="no" checked="false" label="Show advanced options" />
      <when value="yes">
        <param name="is_only" argument="-is_only" type="boolean" truevalue="-is_only" falsevalue="" checked="false" label="Only clip E coli insertion elements" />
        <param name="is_clip" argument="-is_clip" type="boolean" truevalue="-is_clip" falsevalue="" checked="false" label="Clip IS elements before analysis" help="Normally RepeatMasker will report on IS element, with this option selected it will clip them before analysis" />
        <param name="no_is" argument="-no_is" type="boolean" truevalue="-no_is" falsevalue="" checked="false" label="Skip bacterial insertion element check" />
        <param name="rodspec" argument="-rodspec" type="boolean" truevalue="-rodspec" falsevalue="" checked="false" label="Only check for rodent specific repeats" help="If this option is select a check for rodent specific repeats is done instead of a full RepeatMasker run" />
        <param name="primspec" argument="-primspec" type="boolean" truevalue="-primspec" falsevalue="" checked="false" label="Only check for primate specific repeats" help="If this option is select a check for primate specific repeats is done instead of a full RepeatMasker run" />
        <param name="nolow" argument="-nolow" type="boolean" truevalue="-nolow" falsevalue="" checked="false" label="No low complexity masking" help="Skip masking of simple tandem repeats and low complexity regions." />
        <param name="noint" argument="-noint" type="boolean" truevalue="-noint" falsevalue="" checked="false" label="No interspersed repeat masking" help="Only mask simple repeats, skip masking of interspersed repeats." />
        <param name="norna" argument="-norna" type="boolean" truevalue="-norna" falsevalue="" checked="false" label="No repeat-like-RNA masking" help="Skip masking of small pol III transcribed RNA (these are masked by default because they resemble SINEs)" />
        <param name="alu" argument="-alu" type="boolean" truevalue="-alu" falsevalue="" checked="false" label="Limit masking to (primate) Alu repeats" />
        <param name="div" argument="-div" type="boolean" truevalue="-div" falsevalue="" checked="false" label="Limit masking to less diverged (younger) repeats" />
        <param type="select" name="search_speed" label="Search speed vs sensitiviy trade-off">
          <option value="">Default</option>
          <option value="-q">Quick (5-10% less sensitive, 3-4 times speedup)</option>
          <option value="-qq">Rush (10% less sensitive)</option>
          <option value="-s">Slow (0-5% more sensitive, 2.5 times slowdown)</option>
        </param>
        <param name="frag" type="integer" argument="-frag" value="40000" label="Maximum contiguous sequence searched" help="Maximum length of sequencing that is search without fragmenting" />
        <param name="maxsize" type="integer" argument="-maxsize" value="4000000" label="Maximum length for IS or repeat clipped sequences" />
        <param name="gc" type="integer" argument="-gc" value="-1" label="Select matrices for this GC%" help="Valid values are a percentage or -1 to choose the default" />
        <param name="gccalc" type="boolean" argument="-gccalc" truevalue="-gcccalc" falsevalue="" checked="false" label="Calculate GC % for all sequences" help="By default RepeatMasker skips calculating GC % for small sequences" />
        <param name="nocut" type="boolean" argument="-nocut" truevalue="-nocut" falsevalue="" checked="false" label="Skips cutting of repeats" />
        <param name="xout" type="boolean" argument="-x" truevalue="-x" falsevalue="" checked="false" label="Mask with X instead of N characters" />
        <param name="keep_alignments" type="boolean" argument="-ali" truevalue="-ali" falsevalue="" checked="false" label="Output alignments file" />
        <param name="invert_alignments" type="boolean" argument="-inv" truevalue="-inv" falsevalue="" checked="false" label="Invert alignments in alignment file" help="Show alignments in the orientation of the repeat sequence, not the query sequence" />
        <param name="xsmall" type="boolean" argument="-xsmall" truevalue="-xsmall" falsevalue="" checked="false" label="Output repetitive regions as lowercase, non-repetitive regions as uppercase" />
        <param name="poly" type="boolean" argument="-poly" truevalue="-poly" falsevalue="" checked="false" label="Output list of potentially polymorphic microsatellites" />
      </when>
      <when value="no">
      </when>
    </conditional>
  </inputs>
  <outputs>
    <data name="output_masked_genome" format="fasta" label="RepeatMasker masked sequence on ${on_string}">
      <filter>not advanced['show'] or (advanced['show'] and not advanced['is_only'])</filter>
    </data>
    <data name="output_log" format="tabular" label="RepeatMasker output log on ${on_string}">
      <filter>not advanced['show'] or (advanced['show'] and not advanced['is_only'])</filter>
    </data>
    <data name="output_table" format="txt" label="RepeatMasker repeat statistics on ${on_string}">
      <filter>not advanced['show'] or (advanced['show'] and not advanced['is_only'])</filter>
    </data>
    <data name="output_repeat_catalog" format="txt" label="RepeatMasker repeat catalogue on ${on_string}" />
    <data name="output_alignment" format="txt" label="RepeatMasker alignment on ${on_string}">
      <filter>(not advanced['show'] or (advanced['show'] and not advanced['is_only'])) and advanced['show'] and advanced['keep_alignments']</filter>
    </data>
    <data name="output_polymorphic" format="tabular" label="RepeatMasker possible polymorphic repeats on ${on_string}">
      <filter>(not advanced['show'] or (advanced['show'] and not advanced['is_only'])) and advanced['show'] and advanced['poly']</filter>
    </data>
    <data name="output_gff" format="gff" label="RepeatMasker repeat annotation on ${on_string}">
      <filter>(not advanced['show'] or (advanced['show'] and not advanced['is_only'])) and gff is True</filter>
    </data>
  </outputs>
  <tests>
    <test>
      <param name="input_fasta" value="small.fasta" ftype="fasta" />
      <param name="source_type" value="library" />
      <param name="repeat_lib" value="repeats.fasta" ftype="fasta" />
      <output name="output_masked_genome" file="small.fasta.masked" />
      <output name="output_table" file="small.fasta.stats" lines_diff="2" />
      <output name="output_repeat_catalog" file="small.fasta.cat" />
      <output name="output_log" file="small.fasta.log" />
    </test>
    <test>
      <param name="input_fasta" value="small.fasta" ftype="fasta" />
      <param name="source_type" value="library" />
      <param name="gff" value="-gff" />
      <param name="show" value="yes" />
      <param name="keep_alignments" value="-ali" />
      <param name="poly" value="-poly" />
      <param name="repeat_lib" value="repeats.fasta" ftype="fasta" />
      <output name="output_masked_genome" file="small.fasta.masked" />
      <output name="output_table" file="small.fasta.stats" lines_diff="4" />
      <output name="output_repeat_catalog" file="small.fasta.cat" />
      <output name="output_log" file="small.fasta.log" />
      <output name="output_alignment" file="small.fasta.align" />
      <output name="output_polymorphic" file="small.fasta.poly" />
      <output name="output_gff" file="small.fasta.gff" lines_diff="2" />
    </test>
  </tests>
  <help><![CDATA[
RepeatMasker is a program that screens DNA for interspersed repeats and low
complexity DNA sequences. The database of repeats to screen for can be
provided as a FASTA file or downloaded from RepBase_. If the RepBase option is
chosen the RepBaseRepeatMaskerEdition file should be downloaded and
unpacked, and the enclosed EMBL format file ('RMRBSeqs.embl') should
be uploaded to Galaxy for use with this tool.

Further documentation is available on the RepeatMasker homepage_.

.. _RepBase: http://www.girinst.org/repbase/
.. _homepage: http://www.repeatmasker.org/webrepeatmaskerhelp.html
    ]]>
  </help>
  <citations>
    <citation type="bibtex">
      @misc{RepeatMasker,
        title = {RepeatMasker Open-4.0},
        howpublished = {\url{http://www.repeatmasker.org}},
        author = {<NAME> and <NAME> and <NAME>.},
        year = {2013-2015}}
    </citation>
  </citations>
</tool><commit_msg>Accept 4 lines diff for small.fasta.gff<commit_after><tool id="repeat_masker" name="RepeatMasker" version="0.1.3" profile="17.01">
  <description>RepeatMasker</description>

  <requirements>
    <requirement type="package">repeatmasker</requirement>
  </requirements>

  <command detect_errors="exit_code"><![CDATA[
    RM_LIB_PATH=\$(dirname \$(which RepeatMasker))/../share/RepeatMasker/Libraries &&
    mkdir lib &&
    export REPEATMASKER_LIB_DIR=\$(pwd)/lib &&
      for file in \$(ls \$RM_LIB_PATH) ; do  ln -s \$RM_LIB_PATH/\$file lib/\$file ; done &&
    #if $repeat_source.source_type == "repbase":
      cp '${repeat_source.repbase_file}' lib/RMRBSeqs.embl &&
    #end if
    RepeatMasker -dir \$(pwd)
    #if $repeat_source.source_type == "library":
      -lib '${repeat_source.repeat_lib}'
      -cutoff '${repeat_source.cutoff}'
    #else if $repeat_source.source_type == "repbase":
      -species '${repeat_source.species}'
    #end if
    -parallel \${GALAXY_SLOTS:-1}
    '${gff}'
    '${ignore_n_stretches}'
    #if $advanced.show == 'yes':
      '${advanced.is_only}'
      '${advanced.is_clip}'
      '${advanced.no_is}'
      '${advanced.rodspec}'
      '${advanced.primspec}'
      '${advanced.nolow}'
      '${advanced.noint}'
      '${advanced.norna}'
      '${advanced.alu}'
      '${advanced.div}'
      '${advanced.search_speed}'
      '${advanced.frag}'
      '${advanced.maxsize}'
      #if $advanced.gc != -1:
        '${advanced.gc}'
      #end if
      '${advanced.gccalc}'
      '${advanced.nocut}'
      '${advanced.keep_alignments}'
      '${advanced.invert_alignments}'
      '${advanced.xout}'
      '${advanced.xsmall}'
      '${advanced.poly}'
    #end if
    '${input_fasta}' &&
    ls &&
    #if $advanced.show == 'no' or ($advanced.show == 'yes' and $advanced.is_only != '-is_only'):
      mv \$(basename '${input_fasta}').masked '${output_masked_genome}' &&
      sed -r 's/^ *// ; s/ *$//; s/\+ //; s/ +/\t/g ;  1,2c SW score\t% div.\t% del.\t% ins.\tquery sequence\tpos in  query: begin\tend\t(left)\trepeat\tclass/family\tpos in repeat: begin\tend\t(left)\tID' \$(basename '${input_fasta}').out >'${output_log}' &&
      mv \$(basename '${input_fasta}').tbl '${output_table}' &&
      #if $gff == '-gff':
        mv \$(basename '${input_fasta}').out.gff '${output_gff}' &&
      #end if
      #if $advanced.show == 'yes' and $advanced.keep_alignments == '-ali':
        mv \$(basename '${input_fasta}').align '${output_alignment}' &&
      #end if
      #if $advanced.show == 'yes' and $advanced.poly == '-poly':
        sed -r 's/^ *// ; s/ *$//; s/\+ //; s/ +/\t/g' \$(basename '${input_fasta}').polyout >'${output_polymorphic}' &&
      #end if
    #end if
    mv \$(basename '${input_fasta}').cat '${output_repeat_catalog}'
    ]]>
  </command>

  <inputs>
    <param name="input_fasta" type="data" format="fasta" label="Genomic DNA" />
    <conditional name="repeat_source">
      <param label="Repeat library source" name="source_type" type="select">
        <option selected="true" value="repbase">RepBase</option>
        <option value="library">Custom library of repeats</option>
      </param>
      <when value="repbase">
        <param name="repbase_file" type="data" format="embl" label="RepBase (RMRBSeqs.embl) file" />
        <param name="species" type="text" value="homo sapiens" label="Repeat source species" help="Source species (or clade name) used to select repeats from RepBase" />
      </when>
      <when value="library">
        <param name="repeat_lib" type="data" format="fasta" label="Custom library of repeats" />
        <param name="cutoff" type="integer" argument="-cutoff" value="225" label="Cutoff score for masking repeats" />
      </when>
    </conditional>
    <param name="gff" type="boolean" argument="-gff" truevalue="-gff" falsevalue="" label="Output annotation of repeats in GFF format" checked="false" />
    <param name="ignore_n_stretches" type="boolean" argument="-excln" falsevalue="" label="Ignore stretches of Ns when computing statistics" checked="true" help="Scaffolds are sometimes joined with stretches of 25 or more Ns. This option ignores them when calculating repeat statistics" />
    <conditional name="advanced">
      <param name="show" type="boolean" truevalue="yes" falsevalue="no" checked="false" label="Show advanced options" />
      <when value="yes">
        <param name="is_only" argument="-is_only" type="boolean" truevalue="-is_only" falsevalue="" checked="false" label="Only clip E coli insertion elements" />
        <param name="is_clip" argument="-is_clip" type="boolean" truevalue="-is_clip" falsevalue="" checked="false" label="Clip IS elements before analysis" help="Normally RepeatMasker will report on IS element, with this option selected it will clip them before analysis" />
        <param name="no_is" argument="-no_is" type="boolean" truevalue="-no_is" falsevalue="" checked="false" label="Skip bacterial insertion element check" />
        <param name="rodspec" argument="-rodspec" type="boolean" truevalue="-rodspec" falsevalue="" checked="false" label="Only check for rodent specific repeats" help="If this option is select a check for rodent specific repeats is done instead of a full RepeatMasker run" />
        <param name="primspec" argument="-primspec" type="boolean" truevalue="-primspec" falsevalue="" checked="false" label="Only check for primate specific repeats" help="If this option is select a check for primate specific repeats is done instead of a full RepeatMasker run" />
        <param name="nolow" argument="-nolow" type="boolean" truevalue="-nolow" falsevalue="" checked="false" label="No low complexity masking" help="Skip masking of simple tandem repeats and low complexity regions." />
        <param name="noint" argument="-noint" type="boolean" truevalue="-noint" falsevalue="" checked="false" label="No interspersed repeat masking" help="Only mask simple repeats, skip masking of interspersed repeats." />
        <param name="norna" argument="-norna" type="boolean" truevalue="-norna" falsevalue="" checked="false" label="No repeat-like-RNA masking" help="Skip masking of small pol III transcribed RNA (these are masked by default because they resemble SINEs)" />
        <param name="alu" argument="-alu" type="boolean" truevalue="-alu" falsevalue="" checked="false" label="Limit masking to (primate) Alu repeats" />
        <param name="div" argument="-div" type="boolean" truevalue="-div" falsevalue="" checked="false" label="Limit masking to less diverged (younger) repeats" />
        <param type="select" name="search_speed" label="Search speed vs sensitiviy trade-off">
          <option value="">Default</option>
          <option value="-q">Quick (5-10% less sensitive, 3-4 times speedup)</option>
          <option value="-qq">Rush (10% less sensitive)</option>
          <option value="-s">Slow (0-5% more sensitive, 2.5 times slowdown)</option>
        </param>
        <param name="frag" type="integer" argument="-frag" value="40000" label="Maximum contiguous sequence searched" help="Maximum length of sequencing that is search without fragmenting" />
        <param name="maxsize" type="integer" argument="-maxsize" value="4000000" label="Maximum length for IS or repeat clipped sequences" />
        <param name="gc" type="integer" argument="-gc" value="-1" label="Select matrices for this GC%" help="Valid values are a percentage or -1 to choose the default" />
        <param name="gccalc" type="boolean" argument="-gccalc" truevalue="-gcccalc" falsevalue="" checked="false" label="Calculate GC % for all sequences" help="By default RepeatMasker skips calculating GC % for small sequences" />
        <param name="nocut" type="boolean" argument="-nocut" truevalue="-nocut" falsevalue="" checked="false" label="Skips cutting of repeats" />
        <param name="xout" type="boolean" argument="-x" truevalue="-x" falsevalue="" checked="false" label="Mask with X instead of N characters" />
        <param name="keep_alignments" type="boolean" argument="-ali" truevalue="-ali" falsevalue="" checked="false" label="Output alignments file" />
        <param name="invert_alignments" type="boolean" argument="-inv" truevalue="-inv" falsevalue="" checked="false" label="Invert alignments in alignment file" help="Show alignments in the orientation of the repeat sequence, not the query sequence" />
        <param name="xsmall" type="boolean" argument="-xsmall" truevalue="-xsmall" falsevalue="" checked="false" label="Output repetitive regions as lowercase, non-repetitive regions as uppercase" />
        <param name="poly" type="boolean" argument="-poly" truevalue="-poly" falsevalue="" checked="false" label="Output list of potentially polymorphic microsatellites" />
      </when>
      <when value="no">
      </when>
    </conditional>
  </inputs>
  <outputs>
    <data name="output_masked_genome" format="fasta" label="RepeatMasker masked sequence on ${on_string}">
      <filter>not advanced['show'] or (advanced['show'] and not advanced['is_only'])</filter>
    </data>
    <data name="output_log" format="tabular" label="RepeatMasker output log on ${on_string}">
      <filter>not advanced['show'] or (advanced['show'] and not advanced['is_only'])</filter>
    </data>
    <data name="output_table" format="txt" label="RepeatMasker repeat statistics on ${on_string}">
      <filter>not advanced['show'] or (advanced['show'] and not advanced['is_only'])</filter>
    </data>
    <data name="output_repeat_catalog" format="txt" label="RepeatMasker repeat catalogue on ${on_string}" />
    <data name="output_alignment" format="txt" label="RepeatMasker alignment on ${on_string}">
      <filter>(not advanced['show'] or (advanced['show'] and not advanced['is_only'])) and advanced['show'] and advanced['keep_alignments']</filter>
    </data>
    <data name="output_polymorphic" format="tabular" label="RepeatMasker possible polymorphic repeats on ${on_string}">
      <filter>(not advanced['show'] or (advanced['show'] and not advanced['is_only'])) and advanced['show'] and advanced['poly']</filter>
    </data>
    <data name="output_gff" format="gff" label="RepeatMasker repeat annotation on ${on_string}">
      <filter>(not advanced['show'] or (advanced['show'] and not advanced['is_only'])) and gff is True</filter>
    </data>
  </outputs>
  <tests>
    <test>
      <param name="input_fasta" value="small.fasta" ftype="fasta" />
      <param name="source_type" value="library" />
      <param name="repeat_lib" value="repeats.fasta" ftype="fasta" />
      <output name="output_masked_genome" file="small.fasta.masked" />
      <output name="output_table" file="small.fasta.stats" lines_diff="2" />
      <output name="output_repeat_catalog" file="small.fasta.cat" />
      <output name="output_log" file="small.fasta.log" />
    </test>
    <test>
      <param name="input_fasta" value="small.fasta" ftype="fasta" />
      <param name="source_type" value="library" />
      <param name="gff" value="-gff" />
      <param name="show" value="yes" />
      <param name="keep_alignments" value="-ali" />
      <param name="poly" value="-poly" />
      <param name="repeat_lib" value="repeats.fasta" ftype="fasta" />
      <output name="output_masked_genome" file="small.fasta.masked" />
      <output name="output_table" file="small.fasta.stats" lines_diff="4" />
      <output name="output_repeat_catalog" file="small.fasta.cat" />
      <output name="output_log" file="small.fasta.log" />
      <output name="output_alignment" file="small.fasta.align" />
      <output name="output_polymorphic" file="small.fasta.poly" />
      <output name="output_gff" file="small.fasta.gff" lines_diff="4" />
    </test>
  </tests>
  <help><![CDATA[
RepeatMasker is a program that screens DNA for interspersed repeats and low
complexity DNA sequences. The database of repeats to screen for can be
provided as a FASTA file or downloaded from RepBase_. If the RepBase option is
chosen the RepBaseRepeatMaskerEdition file should be downloaded and
unpacked, and the enclosed EMBL format file ('RMRBSeqs.embl') should
be uploaded to Galaxy for use with this tool.

Further documentation is available on the RepeatMasker homepage_.

.. _RepBase: http://www.girinst.org/repbase/
.. _homepage: http://www.repeatmasker.org/webrepeatmaskerhelp.html
    ]]>
  </help>
  <citations>
    <citation type="bibtex">
      @misc{RepeatMasker,
        title = {RepeatMasker Open-4.0},
        howpublished = {\url{http://www.repeatmasker.org}},
        author = {<NAME> and <NAME> <NAME>.},
        year = {2013-2015}}
    </citation>
  </citations>
</tool>
{'fraction_non_alphanumeric': 0.16375615177012223, 'fraction_numerical': 0.004326083505318304, 'mean_word_length': 4.107845124670586, 'pattern_counts': {'":': 6, '<': 269, '<?xml version=': 0, '>': 273, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3293800', 'n_tokens_mistral': 7961, 'n_tokens_neox': 7569, 'n_words': 2039}