<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
This is a skeleton file that can serve as a starting point for a Python
console script. To run this script uncomment the following line in the
entry_points section in setup.cfg:

    console_scripts =
        download = asm_challenge.download:run

Then run `python setup.py install` which will install the command `download`
inside your current environment.
Besides console scripts, the header (i.e. until _logger...) of this file can
also be used as template for Python modules.

Note: This skeleton file can be safely removed if not needed!
"""
from __future__ import division, print_function, absolute_import

import argparse
import sys
import logging
import pandas as pd
import urllib
from StringIO import StringIO
from path import Path
import os
import re
import json

from asm_challenge import __version__

__author__ = "<NAME>"
__copyright__ = "<NAME>"
__license__ = "none"

_logger = logging.getLogger(__name__)


bioproject_files = {
    "295366": "./input/295366.txt",
    "215355": "./input/215355.txt",
    "295367": "./input/295367.txt",
    "237212": "./input/237212.txt",
    "227458": "./input/227458.txt",
    "252015": "./input/252015.txt",
    "230403": "./input/230403.txt"
}

bioproject_organism = {
    "295366": "Salmonella Enteritidis",
    "215355": "Listeria monocytogenes",
    "295367": "Listeria monocytogenes",
    "237212": "Salmonella Enteritidis",
    "227458": "Salmonella Enteritidis",
    "252015": "Salmonella Enteritidis",
    "230403": "Salmonella Enteritidis"
}

def parse_args(args):
    """
    Parse command line parameters

    :param args: command line parameters as list of strings
    :return: command line parameters as :obj:`argparse.Namespace`
    """
    parser = argparse.ArgumentParser(
        description="Just a Hello World demonstration")
    parser.add_argument(
        '-v',
        '--version',
        action='version',
        version='asm_challenge {ver}'.format(ver=__version__))
    return parser.parse_args(args)


def get_fastq_url_from_bioproject(bioproject, dir):
    """
    Get Fastq URK associated with Bioproject from SRA archive

    :param bioproject: ABioproject from SRA
    :return: table of FTP URLs
    """

    # base_url_old = 'http://www.ncbi.nlm.nih.gov/sra?'\
    #                'linkname=bioproject_sra_all&'\
    #                'from_uid='
    url = 'http://www.ncbi.nlm.nih.gov/biosample?'\
          'LinkName=bioproject_biosample_all&from_uid=%s'\
          '&format=text' % (bioproject)
    data = urllib.urlopen(url).read()
    m = re.findall(r"SRA: (.+)", data)
    for accession in m:
        _logger.info(accession)
        download_fastqSRA(accession, dir)

def get_metadata(accession, bioID):

    metadata = {
        "sample_name": accession,
        "group_name": "",
        "file_names": "",
        "sequencing_platform": "Illumina",
        "sequencing_type": "single",
        "pre_assembled": "no",
        "sample_type": "isolate",
        "organism": bioproject_organism[bioID],
        "strain": "",
        "subtype": "",
        "country": "",
        "region": "",
        "city": "",
        "zip_code": "",
        "longitude": "",
        "latitude": "",
        "location_note": "",
        "isolation_source": "other",
        "source_note": "",
        "pathogenic": "unknown",
        "pathogenicity_note": "",
        "collection_date": "",
        "collected_by": "",
        "usage_restrictions": "private",
        "release_date": "",
        "email_address": "",
        "notes": "",
        "upload_dir": 1,
        "batch": "true"
    }
    isolation_source = ["human", "water", "food", "animal", "laboratory", "other"]
    url = 'http://www.ncbi.nlm.nih.gov/sra/?term='+ accession +'&format=text'
    data = urllib.urlopen(url).read()
    m = re.findall(r"Sample Attributes: (.+)\n", data)
    _logger.info(m)
    for answer in m:
        for attributes in answer.split(';'):
            stat = attributes.split('=')
            _logger.info(stat)
            if stat[0].strip() == 'geo_loc_name' and ":" in stat[1]:
                metadata["country"] = stat[1].split(":")[0]
                metadata["region"] = stat[1].split(":")[1]
            elif stat[0].strip() == 'geo_loc_name':
                metadata["country"] = stat[1]
            if stat[0].strip() in metadata:
                if stat[0].strip() == 'isolation_source':
                    if stat[1] not in isolation_source:
                        metadata[stat[0].strip()] = 'other'
                        metadata['source_note'] = stat[1]
                elif stat[0].strip() == 'BioSample':
                    metadata['notes'] = metadata['notes'] + ' BioSample=' + stat[1] + ', '
                # elif stat[0].strip() == 'run':
                #     metadata['notes'] = metadata['notes'] + ' run=' + stat[1]  + ', '
                else:
                    metadata[stat[0].strip()] = stat[1]
        # metadata['file_names'] = '%s_1.fastq.gz %s_2.fastq.gz' % (accession.strip('\n'), accession.strip('\n'))
        metadata['file_names'] = '%s.fastq' % (accession.strip('\n'))<commit_msg>Splitted files metadata added<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
This is a skeleton file that can serve as a starting point for a Python
console script. To run this script uncomment the following line in the
entry_points section in setup.cfg:

    console_scripts =
        download = asm_challenge.download:run

Then run `python setup.py install` which will install the command `download`
inside your current environment.
Besides console scripts, the header (i.e. until _logger...) of this file can
also be used as template for Python modules.

Note: This skeleton file can be safely removed if not needed!
"""
from __future__ import division, print_function, absolute_import

import argparse
import sys
import logging
import pandas as pd
import urllib
from StringIO import StringIO
from path import Path
import os
import re
import json

from asm_challenge import __version__

__author__ = "<NAME>"
__copyright__ = "<NAME>"
__license__ = "none"

_logger = logging.getLogger(__name__)


bioproject_files = {
    "295366": "./input/295366.txt",
    "215355": "./input/215355.txt",
    "295367": "./input/295367.txt",
    "237212": "./input/237212.txt",
    "227458": "./input/227458.txt",
    "252015": "./input/252015.txt",
    "230403": "./input/230403.txt"
}

bioproject_organism = {
    "295366": "Salmonella Enteritidis",
    "215355": "Listeria monocytogenes",
    "295367": "Listeria monocytogenes",
    "237212": "Salmonella Enteritidis",
    "227458": "Salmonella Enteritidis",
    "252015": "Salmonella Enteritidis",
    "230403": "Salmonella Enteritidis"
}

def parse_args(args):
    """
    Parse command line parameters

    :param args: command line parameters as list of strings
    :return: command line parameters as :obj:`argparse.Namespace`
    """
    parser = argparse.ArgumentParser(
        description="Just a Hello World demonstration")
    parser.add_argument(
        '-v',
        '--version',
        action='version',
        version='asm_challenge {ver}'.format(ver=__version__))
    return parser.parse_args(args)


def get_fastq_url_from_bioproject(bioproject, dir):
    """
    Get Fastq URK associated with Bioproject from SRA archive

    :param bioproject: ABioproject from SRA
    :return: table of FTP URLs
    """

    # base_url_old = 'http://www.ncbi.nlm.nih.gov/sra?'\
    #                'linkname=bioproject_sra_all&'\
    #                'from_uid='
    url = 'http://www.ncbi.nlm.nih.gov/biosample?'\
          'LinkName=bioproject_biosample_all&from_uid=%s'\
          '&format=text' % (bioproject)
    data = urllib.urlopen(url).read()
    m = re.findall(r"SRA: (.+)", data)
    for accession in m:
        _logger.info(accession)
        download_fastqSRA(accession, dir)

def get_metadata(accession, bioID):

    metadata = {
        "sample_name": accession,
        "group_name": "",
        "file_names": "",
        "sequencing_platform": "Illumina",
        "sequencing_type": "single",
        "pre_assembled": "no",
        "sample_type": "isolate",
        "organism": bioproject_organism[bioID],
        "strain": "",
        "subtype": "",
        "country": "",
        "region": "",
        "city": "",
        "zip_code": "",
        "longitude": "",
        "latitude": "",
        "location_note": "",
        "isolation_source": "other",
        "source_note": "",
        "pathogenic": "unknown",
        "pathogenicity_note": "",
        "collection_date": "",
        "collected_by": "",
        "usage_restrictions": "private",
        "release_date": "",
        "email_address": "",
        "notes": "",
        "upload_dir": 1,
        "batch": "true"
    }
    isolation_source = ["human", "water", "food", "animal", "laboratory", "other"]
    url = 'http://www.ncbi.nlm.nih.gov/sra/?term='+ accession +'&format=text'
    data = urllib.urlopen(url).read()
    m = re.findall(r"Sample Attributes: (.+)\n", data)
    _logger.info(m)
    for answer in m:
        for attributes in answer.split(';'):
            stat = attributes.split('=')
            _logger.info(stat)
            if stat[0].strip() == 'geo_loc_name' and ":" in stat[1]:
                metadata["country"] = stat[1].split(":")[0]
                metadata["region"] = stat[1].split(":")[1]
            elif stat[0].strip() == 'geo_loc_name':
                metadata["country"] = stat[1]
            if stat[0].strip() in metadata:
                if stat[0].strip() == 'isolation_source':
                    if stat[1] not in isolation_source:
                        metadata[stat[0].strip()] = 'other'
                        metadata['source_note'] = stat[1]
                elif stat[0].strip() == 'BioSample':
                    metadata['notes'] = metadata['notes'] + ' BioSample=' + stat[1] + ', '
                # elif stat[0].strip() == 'run':
                #     metadata['notes'] = metadata['notes'] + ' run=' + stat[1]  + ', '
                else:
                    metadata[stat[0].strip()] = stat[1]
        metadata['file_names'] = '%s_1.fastq.gz %s_2.fastq.gz' % (accession.strip('\n'), accession.strip('\n'))
        # metadata['file_names'] = '%s.fastq' % (accession.strip('\n'))
{'fraction_non_alphanumeric': 0.155155736119172, 'fraction_numerical': 0.028825691623137938, 'mean_word_length': 2.5824670824670823, 'pattern_counts': {'":': 92, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2940591', 'n_tokens_mistral': 3510, 'n_tokens_neox': 3176, 'n_words': 750}