<commit_before><tool id="retrieve_fasta_from_NCBI" name="Retrieve FASTA from NCBI" version="2.2.1">
  <description></description>
  <command><![CDATA[
    python '$__tool_directory__'/fetch_fasta_from_NCBI.py
        -i "$queryString"
        -d $dbname
        -l '$logfile'
        -c
        -o '$outfile';
    #if $dry_run == ""
        number_UIDs=\$(tail -n 2 $logfile | perl -ne '/Found (\d+) UID/ && print \$1');
        if [ "\$number_UIDs" -lt "300000" ]; then
            python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                -i "$queryString"
                -d $dbname
                -l '$logfile'
                -o '$outfile';
        else
            python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                -i "$queryString"
                -d $dbname
                -u
                -l '$logfile'
                -o 'uid_outfile';
            UID_array=( \$(head uid_outfile) );
            array_len=\${#UID_array[@]};
            counter=0;
            echo "\$array_len";
            if [ "\$array_len" -gt "200000" ]; then
                number_of_groups=\$((array_len / 200000));
                modulo=\$((array_len % 200000));
                if [ "\$modulo" -gt 0 ];then
                    number_of_groups=\$((number_of_groups + 1));
                fi;
                group_number=1;
                echo "----- Number of groups of batches: \$number_of_groups -----" >> $logfile;
                for ((i=0; i+200000<array_len;i+=200000)); do
                    echo "----- Group number: \$group_number -----" >> $logfile;
                    python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                        -d $dbname
                        -l '$logfile'
                        -o 'tmp1_outfile'
                        --UID_list \${UID_array[@]:\$i:99999}&
                    python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                        -d $dbname
                        -l 'tmp1_logfile'
                        -o 'tmp2_outfile'
                        --UID_list \${UID_array[@]:\$((i+100000)):99999}&
              #*      python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                        -d $dbname
                        -l 'tmp2_logfile'
                        -o 'tmp3_outfile'
                        --UID_list \${UID_array[@]:\$((i+200000)):99999}&
                    python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                        -d $dbname
                        -l 'tmp3_logfile'
                        -o 'tmp4_outfile'
                        --UID_list \${UID_array[@]:\$((i+300000)):99999}&*#
                    wait;
                    cat tmp1_outfile tmp2_outfile >> $outfile;
                    rm tmp1_outfile tmp2_outfile;
                    cat tmp1_logfile >> $logfile;
                    rm tmp1_logfile;
                    group_number=\$((group_number + 1));
                    counter=\$(( counter + 200000 ));
                done;
                echo "----- Group number: \$group_number -----" >> $logfile;
                echo "----- Last group -----" >> $logfile;
                if [ "\$counter" -lt "\$array_len" ]; then
                python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                    -d $dbname
                    -l '$logfile'
                    -o '$outfile'
                    --UID_list \${UID_array[@]:\$counter:\$((array_len - counter + 1))};
                fi;
            else
                python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                    -d $dbname
                    -l '$logfile'
                    -o '$outfile'
                    --UID_list \${UID_array[@]:0:\$array_len};
            fi;
        fi;
    #end if
  ]]></command>

  <inputs>
    <param name="queryString" type="text" size="5x80" area="True" value="txid10239[orgn] NOT txid131567[orgn] AND complete[all] NOT partial[title] NOT phage[title]" label="Query to NCBI in entrez format" help="exemple:'Drosophila melanogaster[Organism] AND Gcn5[Title]">
      <sanitizer>
        <valid initial="string.printable">
          <remove value="&quot;"/>
          <remove value="\"/>
        </valid>
        <mapping initial="none">
          <add source="&quot;" target="\&quot;"/>
          <add source="\" target="\\"/>
        </mapping>
      </sanitizer><commit_msg>Modified xml so as to pass a file of UIDs. Corrected error in last group loop added temporary output file so as to not overrite the previous sequences<commit_after><tool id="retrieve_fasta_from_NCBI" name="Retrieve FASTA from NCBI" version="2.2.1">
  <description></description>
  <command><![CDATA[
    python '$__tool_directory__'/fetch_fasta_from_NCBI.py
        -i "$queryString"
        -d $dbname
        -l '$logfile'
        -c
        -o '$outfile';
    #if $dry_run == ""
        number_UIDs=\$(tail -n 2 $logfile | perl -ne '/Found (\d+) UID/ && print \$1');
        if [ "\$number_UIDs" -lt "300000" ]; then
            python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                -i "$queryString"
                -d $dbname
                -l '$logfile'
                -o '$outfile';
        else
            python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                -i "$queryString"
                -d $dbname
                -u
                -l '$logfile'
                -o 'uid_outfile';
            UID_array=( \$(head uid_outfile) );
            array_len=\${#UID_array[@]};
            counter=0;
            if [ "\$array_len" -gt "400000" ]; then
                number_of_groups=\$((array_len / 400000));
                modulo=\$((array_len % 400000));
                if [ "\$modulo" -gt 0 ];then
                    number_of_groups=\$((number_of_groups + 1));
                fi;
                group_number=1;
                echo "----- Number of groups of batches: \$number_of_groups -----" >> $logfile;
                for ((i=0; i+400000<array_len;i+=400000)); do
                    echo "----- Group number: \$group_number -----" >> $logfile;
                    echo "\${UID_array[@]:\$i:99999}" > uid_list_1.txt;
                    echo "\${UID_array[@]:\$((i+100000)):99999}" > uid_list_2.txt;
                    echo "\${UID_array[@]:\$((i+200000)):99999}" > uid_list_3.txt;
                    echo "\${UID_array[@]:\$((i+300000)):99999}" > uid_list_4.txt;
                    python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                        -d $dbname
                        -l '$logfile'
                        -o 'tmp1_outfile'
                        --UID_list uid_list_1.txt&
                    python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                        -d $dbname
                        -l 'tmp1_logfile'
                        -o 'tmp2_outfile'
                        --UID_list uid_list_2.txt&
                    python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                        -d $dbname
                        -l 'tmp2_logfile'
                        -o 'tmp3_outfile'
                        --UID_list uid_list_3.txt&
                    python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                        -d $dbname
                        -l 'tmp3_logfile'
                        -o 'tmp4_outfile'
                        --UID_list &
                    wait;
                    cat tmp1_outfile tmp2_outfile tmp3_outfile tmp4_outfile >> $outfile;
                    rm tmp1_outfile tmp2_outfile tmp3_outfile tmp4_outfile;
                    cat tmp1_logfile tmp2_logfile tmp3_logfile >> $logfile;
                    rm tmp1_logfile tmp2_logfile tmp3_logfile;
                    rm uid_list_1.txt uid_list_2.txt uid_list_3.txt uid_list_4.txt;
                    group_number=\$((group_number + 1));
                    counter=\$(( counter + 400000 ));
                done;
                echo "----- Group number: \$group_number -----" >> $logfile;
                echo "----- Last group -----" >> $logfile;
                if [ "\$counter" -lt "\$array_len" ]; then
                echo "\${UID_array[@]:\$counter:\$((array_len - counter + 1))}" > uid_list.txt;
                python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                    -d $dbname
                    -l '$logfile'
                    -o 'tmp_outfile'
                    --UID_list uid_list.txt;
                rm uid_list.txt;
                cat tmp_outfile >> $outfile;
                rm tmp_outfile;
                fi;
            else
                echo "\${UID_array[@]:0:\$array_len}" > uid_list.txt;
                python '$__tool_directory__'/fetch_fasta_from_NCBI.py
                    -d $dbname
                    -l '$logfile'
                    -o '$outfile'
                    --UID_list uid_list.txt;
                rm uid_list.txt;
            fi;
        fi;
    #end if
  ]]></command>

  <inputs>
    <param name="queryString" type="text" size="5x80" area="True" value="txid10239[orgn] NOT txid131567[orgn] AND complete[all] NOT partial[title] NOT phage[title]" label="Query to NCBI in entrez format" help="exemple:'Drosophila melanogaster[Organism] AND Gcn5[Title]">
      <sanitizer>
        <valid initial="string.printable">
          <remove value="&quot;"/>
          <remove value="\"/>
        </valid>
        <mapping initial="none">
          <add source="&quot;" target="\&quot;"/>
          <add source="\" target="\\"/>
        </mapping>
      </sanitizer>
{'fraction_non_alphanumeric': 0.13735167489236585, 'fraction_numerical': 0.027407329623017958, 'mean_word_length': 1.335458558116724, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5007787', 'n_tokens_mistral': 3089, 'n_tokens_neox': 2749, 'n_words': 557}