<commit_before>#!/bin/bash
#
# get-dra.sh ver 0.1.0 by <EMAIL>
# usable only in DDBJ supercomputer system
#
# usage:
#   get-dra <SRA Experiment ID> <Output directory>
#
set -eu

#
# functions
#

connect_dra(){
  files_in_dra=`ssh t347 ls /usr/local/ftp/public/ddbj_database/dra 2> /dev/null` &&:
  if [[ -z "${files_in_dra}" ]] ; then
    echo "Cannot connect to DRA node: check your ssh configuration"
    exit 1
  fi
}

get_submission_id(){
  local exp_id=${1}

  # retrieve accession table if local file is not found
  if [[ ! -e "/home/`id -nu`/.dra/latest/SRA_Accessions.tab" ]] ; then
    update_accession_table
  fi

  # extract id
  local sub_id=`awk_extract_submission_id`

  # id cannot be found if accession tabel is old: update and try again
  if [[ -z "${sub_id}" ]] ; then
    update_accession_table
    local sub_id=`awk_extract_submission_id`
  fi

  echo "${sub_id}"
}

update_accession_table(){
  local homedir="/home/`id -nu`"
  # Create directory for SRA Accession table
  local latest_dir="${homedir}/.dra/latest"
  mkdir -p "${latest_dir}"

  # define required tables
  local accessions="${latest_dir}/SRA_Accessions.tab"
  local fastqlist="${latest_dir}/fastqlist"
  local sralist="${latest_dir}/sralist"

  # Move old accesison tables
  if [[ -e "${accessions}" ]] ; then
    backup_dir="${homedir}/.dra/"`date "+%Y%m%d"`
    mkdir -p "${backup_dir}"
    mv "${accessions}" "${backup_dir}"
    mv "${fastqlist}" "${backup_dir}"
    mv "${sralist}" "${backup_dir}"
  fi

  # retrieve from NCBI ftp
  `lftp -c "open ftp.ncbi.nlm.nih.gov:/sra/reports/Metadata && pget -O ${latest_dir} -n 8 SRA_Accessions.tab"`

  # retrieve from DDBJ ftp
  `lftp -c "open ftp.ddbj.nig.ac.jp/ddbj_database/dra/meta/list && get -O ${latest_dir} fastqlist && get -O ${latest_dir} sralist"`
}

awk_extract_submission_id(){
  cat "/home/`id -nu`/.dra/latest/SRA_Accessions.tab" | awk -F '\t' --assign id="${exp_id}" '$1 == id { print $2 }'
}

get_filepath(){
  local exp_id=${1}

  # try to get path to fastq file
  local fpath=`get_fq_path "${exp_id}"`

  # get path to sra file if fastq file not found
  if [[ -z "${fpath}" ]] ; then
    local fpath=`get_sra_path "${exp_id}"`
  fi

  # return filepath
  echo "${fpath}"
}

get_fq_path(){
  local exp_id=${1}

  # retrieve submission id
  local sub_id=`get_submission_id "${exp_id}"`
  local sub_id_head=`echo ${sub_id:0:6}`

  # path to fastq under ddbj_database/dra directory
  local fq_path="fastq/${sub_id_head}/${sub_id}/${exp_id}"

  # path to ddbj_database/dra on local filesystem
  local dra_path="/usr/local/ftp/public/ddbj_database/dra"

  # check if file is available
  local fq_list=`ssh t347 ls -lk "${dra_path}/${fq_path}"`
  if [[ ! -z "${fq_list}" ]] ; then
    echo "${fq_path}"
  fi
}

get_sra_path(){
  local exp_id=${1}
  local sra_path="sralite/ByExp/litesra/${exp_id:0:3}/${exp_id:0:6}/${exp_id}"
  echo "${sra_path}"
}

retrieve(){
  local exp_id=${1}
  local path=${2}
  local outdir=${3}
  local ftp_connection_log_dir="/home/`id -nu`/.dra/ftp/"

  # put a file in connection dir to avoid making multiple ftp connections
  queuing_connection "${exp_id}" "${path}"

  # connect to ftp server and change status from waiting to connected, then mirror files
  local ftp_base="ftp.ddbj.nig.ac.jp/ddbj_database/dra"
  lftp -c "open ${ftp_base} && (!mv ${ftp_connection_log_dir}/${exp_id}.waiting ${ftp_connection_log_dir}/${exp_id}.connected) && mirror ${path} ${outdir}"

  # remove file from connection dir
  leave_from_queue
}

queuing_connection(){
  local exp_id=${1}
  local fpath=${2}

  # initialize connection log dir<commit_msg>removed unnecessary slash in the path<commit_after>#!/bin/bash
#
# get-dra.sh ver 0.1.0 by <EMAIL>
# usable only in DDBJ supercomputer system
#
# usage:
#   get-dra <SRA Experiment ID> <Output directory>
#
set -eu

#
# functions
#

connect_dra(){
  files_in_dra=`ssh t347 ls /usr/local/ftp/public/ddbj_database/dra 2> /dev/null` &&:
  if [[ -z "${files_in_dra}" ]] ; then
    echo "Cannot connect to DRA node: check your ssh configuration"
    exit 1
  fi
}

get_submission_id(){
  local exp_id=${1}

  # retrieve accession table if local file is not found
  if [[ ! -e "/home/`id -nu`/.dra/latest/SRA_Accessions.tab" ]] ; then
    update_accession_table
  fi

  # extract id
  local sub_id=`awk_extract_submission_id`

  # id cannot be found if accession tabel is old: update and try again
  if [[ -z "${sub_id}" ]] ; then
    update_accession_table
    local sub_id=`awk_extract_submission_id`
  fi

  echo "${sub_id}"
}

update_accession_table(){
  local homedir="/home/`id -nu`"
  # Create directory for SRA Accession table
  local latest_dir="${homedir}/.dra/latest"
  mkdir -p "${latest_dir}"

  # define required tables
  local accessions="${latest_dir}/SRA_Accessions.tab"
  local fastqlist="${latest_dir}/fastqlist"
  local sralist="${latest_dir}/sralist"

  # Move old accesison tables
  if [[ -e "${accessions}" ]] ; then
    backup_dir="${homedir}/.dra/"`date "+%Y%m%d"`
    mkdir -p "${backup_dir}"
    mv "${accessions}" "${backup_dir}"
    mv "${fastqlist}" "${backup_dir}"
    mv "${sralist}" "${backup_dir}"
  fi

  # retrieve from NCBI ftp
  `lftp -c "open ftp.ncbi.nlm.nih.gov:/sra/reports/Metadata && pget -O ${latest_dir} -n 8 SRA_Accessions.tab"`

  # retrieve from DDBJ ftp
  `lftp -c "open ftp.ddbj.nig.ac.jp/ddbj_database/dra/meta/list && get -O ${latest_dir} fastqlist && get -O ${latest_dir} sralist"`
}

awk_extract_submission_id(){
  cat "/home/`id -nu`/.dra/latest/SRA_Accessions.tab" | awk -F '\t' --assign id="${exp_id}" '$1 == id { print $2 }'
}

get_filepath(){
  local exp_id=${1}

  # try to get path to fastq file
  local fpath=`get_fq_path "${exp_id}"`

  # get path to sra file if fastq file not found
  if [[ -z "${fpath}" ]] ; then
    local fpath=`get_sra_path "${exp_id}"`
  fi

  # return filepath
  echo "${fpath}"
}

get_fq_path(){
  local exp_id=${1}

  # retrieve submission id
  local sub_id=`get_submission_id "${exp_id}"`
  local sub_id_head=`echo ${sub_id:0:6}`

  # path to fastq under ddbj_database/dra directory
  local fq_path="fastq/${sub_id_head}/${sub_id}/${exp_id}"

  # path to ddbj_database/dra on local filesystem
  local dra_path="/usr/local/ftp/public/ddbj_database/dra"

  # check if file is available
  local fq_list=`ssh t347 ls -lk "${dra_path}/${fq_path}"`
  if [[ ! -z "${fq_list}" ]] ; then
    echo "${fq_path}"
  fi
}

get_sra_path(){
  local exp_id=${1}
  local sra_path="sralite/ByExp/litesra/${exp_id:0:3}/${exp_id:0:6}/${exp_id}"
  echo "${sra_path}"
}

retrieve(){
  local exp_id=${1}
  local path=${2}
  local outdir=${3}
  local ftp_connection_log_dir="/home/`id -nu`/.dra/ftp"

  # put a file in connection dir to avoid making multiple ftp connections
  queuing_connection "${exp_id}" "${path}"

  # connect to ftp server and change status from waiting to connected, then mirror files
  local ftp_base="ftp.ddbj.nig.ac.jp/ddbj_database/dra"
  lftp -c "open ${ftp_base} && (!mv ${ftp_connection_log_dir}/${exp_id}.waiting ${ftp_connection_log_dir}/${exp_id}.connected) && mirror ${path} ${outdir}"

  # remove file from connection dir
  leave_from_queue
}

queuing_connection(){
  local exp_id=${1}
  local fpath=${2}

  # initialize connection log dir
{'fraction_non_alphanumeric': 0.15891205301670577, 'fraction_numerical': 0.008007731602926964, 'mean_word_length': 3.9582477754962353, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '711423', 'n_tokens_mistral': 2893, 'n_tokens_neox': 2706, 'n_words': 680}