<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''

'''

from source import ontology, platforms, location_hash
from template import metadata, default
import re
import urllib
import copy
import logging
import sys
import json
import io

logging.basicConfig(
    level=logging.INFO,
    stream=sys.stdout,
    format='%(levelname)s:%(message)s',
    filename='metadata.log',
    filemode='w'
)
_logger = logging.getLogger(__name__)


class Metadata(object):

    def __init__(self, accession, json=default):
        self.metadata = copy.deepcopy(metadata)
        self.metadata.update(json["seed"])
        if self.metadata['sample_name'] == 'ACCESSION':
            self.metadata['sample_name'] = accession.strip()
        self.url = 'http://www.ncbi.nlm.nih.gov/sra/?term=%s&format=text' % (
            accession.strip())
        self.data = urllib.urlopen(self.url).read()
        self.mandatory = json['mandatory']
        self.sample_accession = ''
        self.accession = ''

    def valid_metadata(self):
        '''
        Checks if metadata is valid
        :return: True if all mandatory fields are not ''
        '''
        for i in self.mandatory:
            if self.metadata[i] == '':
                return False
                break
        return True

    def save_metadata(self, dir):
        '''
        Writes self.metadata as meta.json in dir
        :param: dir
        :return: True
        '''
        f = open('%s/meta.json' % dir, 'w')
        f.write(json.dumps(self.metadata, ensure_ascii=False))
        f.close()
        return True

    def update_files(self, files):
        '''
        Checks if metadata is valid
        :return: True if all mandatory fields are not ''
        '''
        self.metadata['files_names'] = files

    def __format_date(yyyy=None, mm=None, dd=None):
        '''
        This method stringify the date tuple using a standard format:
          YYYY-MM-DD or
          YYYY-MM or
          YYYY
        If all arguments are None, FormatDate returns an empty string
        '''
        date = ''
        if yyyy is not None and yyyy != '':
            if mm is not None and mm != '':
                if dd is not None and dd != '':
                    date = '%04d-%02d-%02d' % (yyyy, mm, dd)
                else:
                    date = '%04d-%02d' % (yyyy, mm)
            else:
                date = '%04d' % (yyyy)
        return date

    def __interpret_date(val):
        '''
        This function will try to interpret the
        :param: val Date
        :return: Tuple of integers: (yyyy, mm, dd) or (None, None, None) if
            value not identified
            >>> InterpretDate('Feb-30-2014')
            (2014, 2, 30)
            >>> InterpretDate('Feb-2014')
            ('2014', 2, None)
        '''
        month = 'Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split()
        yyyy, mm, dd = None, None, None
        if '/' in val:
            # American style
            try:
                mm, dd, yyyy = val.split('/')
            except:
                pass
        elif '-' in val:
            # European style
            tmp = val.split('-')
            lens = [len(x) for x in tmp]
            lenl = len(lens)
            if lenl == 3:
                if lens[2] == 4:
                    yyyy = tmp[2]
                    if tmp[1] in month and lens[0] <= 2:
                        mm = month.index(tmp[1]) + 1
                        dd = tmp[0]
                    elif tmp[0] in month and lens[1] <= 2:
                        mm = month.index(tmp[0]) + 1
                        dd = tmp[1]
                    elif lens[0] <= 2 and lens[1] <= 2:
                        mm = tmp[1]
                        dd = tmp[0]
                elif lens[0] == 4:
                    yyyy = tmp[0]
                    if tmp[1] in month and lens[2] <= 2:
                        mm = month.index(tmp[1]) + 1
                        dd = tmp[2]
                    elif lens[1] <= 2 and lens[2] <= 2:
                        mm = tmp[1]
                        dd = tmp[2]
                elif lens[0] <= 2 and lens[2] == 2 and tmp[1] in month:
                    yyyy, mm, dd = '20' + \
                        tmp[2], month.index(tmp[1]) + 1, tmp[0]
            elif lenl == 2:
                if lens[0] == 4:
                    yyyy = tmp[0]
                    if tmp[1] in month:
                        mm = month.index(tmp[1]) + 1
                    elif lens[1] <= 2:
                        mm = tmp[1]
                if lens[1] == 4:
                    yyyy = tmp[1]
                    if tmp[0] in month:
                        mm = month.index(tmp[0]) + 1
                    elif lens[0] <= 2:
                        mm = tmp[0]
                elif lens[1] == 2 and tmp[0] in month:
                    yyyy, mm = '20' + tmp[1], month.index(tmp[0]) + 1
                elif lens[0] == 2 and tmp[1] in month:
                    yyyy, mm = '20' + tmp[0], month.index(tmp[1]) + 1
        elif len(val) == 4:
            yyyy = val
        return (int(yyyy) if yyyy is not None else None,
                int(mm) if mm is not None else None,
                int(dd) if dd is not None else None)

    def update_attributes(self):
        '''
        XXX
        :return: accessionid of non identified sources
        '''
        match = re.findall(r'Run #1: (.+)\n', self.data)
        for answer in match:
            self.accession = answer.split(',')[0]

        match = re.findall(r'Sample Attributes: (.+)\n', self.data)
        for answer in match:
            for attributes in answer.split(';'):
                stat = attributes.split('=')
                att = stat[0].strip('/ ').lower().replace('\'', '')
                val = stat[1].strip('\' ').replace('\'', '\`')
                if att == 'geo_loc_name' and ':' in stat[1]:
                    self.metadata['country'] = val.split(':')[0]
                    self.metadata['region'] = val.split(':')[1]
                elif att == 'geo_loc_name':
                    self.metadata['country'] = val
                elif att == 'serovar':
                    self.metadata['subtype']['serovar'] = val
                elif att == 'mlst':
                    self.metadata['subtype']['mlst'] = val<commit_msg>FIXME added to metadata class:<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''

'''

from source import ontology, platforms, location_hash
from template import metadata, default
import re
import urllib
import copy
import logging
import sys
import json
import io

logging.basicConfig(
    level=logging.INFO,
    stream=sys.stdout,
    format='%(levelname)s:%(message)s',
    filename='metadata.log',
    filemode='w'
)
_logger = logging.getLogger(__name__)


class Metadata(object):

    def __init__(self, accession, json=default):
        self.metadata = copy.deepcopy(metadata)
        self.metadata.update(json["seed"])
        if self.metadata['sample_name'] == 'ACCESSION':
            self.metadata['sample_name'] = accession.strip()
        self.url = 'http://www.ncbi.nlm.nih.gov/sra/?term=%s&format=text' % (
            accession.strip())
        self.data = urllib.urlopen(self.url).read()
        self.mandatory = json['mandatory']
        self.sample_accession = ''
        self.accession = ''

    def valid_metadata(self):
        '''
        Checks if metadata is valid
        :return: True if all mandatory fields are not ''
        '''
        for i in self.mandatory:
            if self.metadata[i] == '':
                return False
                break
        return True

    def save_metadata(self, dir):
        '''
        Writes self.metadata as meta.json in dir
        :param: dir
        :return: True
        '''
        f = open('%s/meta.json' % dir, 'w')
        f.write(json.dumps(self.metadata, ensure_ascii=False))
        f.close()
        return True

    def update_files(self, files):
        '''
        Checks if metadata is valid
        :return: True if all mandatory fields are not ''
        '''
        self.metadata['files_names'] = files

    def __format_date(yyyy=None, mm=None, dd=None):
        '''
        This method stringify the date tuple using a standard format:
          YYYY-MM-DD or
          YYYY-MM or
          YYYY
        If all arguments are None, FormatDate returns an empty string
        '''
        date = ''
        if yyyy is not None and yyyy != '':
            if mm is not None and mm != '':
                if dd is not None and dd != '':
                    date = '%04d-%02d-%02d' % (yyyy, mm, dd)
                else:
                    date = '%04d-%02d' % (yyyy, mm)
            else:
                date = '%04d' % (yyyy)
        return date

    def __interpret_date(val):
        '''
        This function will try to interpret the
        :param: val Date
        :return: Tuple of integers: (yyyy, mm, dd) or (None, None, None) if
            value not identified
            >>> InterpretDate('Feb-30-2014')
            (2014, 2, 30)
            >>> InterpretDate('Feb-2014')
            ('2014', 2, None)
        '''
        month = 'Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split()
        yyyy, mm, dd = None, None, None
        if '/' in val:
            # American style
            try:
                mm, dd, yyyy = val.split('/')
            except:
                pass
        elif '-' in val:
            # European style
            tmp = val.split('-')
            lens = [len(x) for x in tmp]
            lenl = len(lens)
            if lenl == 3:
                if lens[2] == 4:
                    yyyy = tmp[2]
                    if tmp[1] in month and lens[0] <= 2:
                        mm = month.index(tmp[1]) + 1
                        dd = tmp[0]
                    elif tmp[0] in month and lens[1] <= 2:
                        mm = month.index(tmp[0]) + 1
                        dd = tmp[1]
                    elif lens[0] <= 2 and lens[1] <= 2:
                        mm = tmp[1]
                        dd = tmp[0]
                elif lens[0] == 4:
                    yyyy = tmp[0]
                    if tmp[1] in month and lens[2] <= 2:
                        mm = month.index(tmp[1]) + 1
                        dd = tmp[2]
                    elif lens[1] <= 2 and lens[2] <= 2:
                        mm = tmp[1]
                        dd = tmp[2]
                elif lens[0] <= 2 and lens[2] == 2 and tmp[1] in month:
                    yyyy, mm, dd = '20' + \
                        tmp[2], month.index(tmp[1]) + 1, tmp[0]
            elif lenl == 2:
                if lens[0] == 4:
                    yyyy = tmp[0]
                    if tmp[1] in month:
                        mm = month.index(tmp[1]) + 1
                    elif lens[1] <= 2:
                        mm = tmp[1]
                if lens[1] == 4:
                    yyyy = tmp[1]
                    if tmp[0] in month:
                        mm = month.index(tmp[0]) + 1
                    elif lens[0] <= 2:
                        mm = tmp[0]
                elif lens[1] == 2 and tmp[0] in month:
                    yyyy, mm = '20' + tmp[1], month.index(tmp[0]) + 1
                elif lens[0] == 2 and tmp[1] in month:
                    yyyy, mm = '20' + tmp[0], month.index(tmp[1]) + 1
        elif len(val) == 4:
            yyyy = val
        return (int(yyyy) if yyyy is not None else None,
                int(mm) if mm is not None else None,
                int(dd) if dd is not None else None)

    def update_attributes(self):
        '''
        XXX
        :return: accessionid of non identified sources
        '''
        # FIXME: Major problem:
        #   sequencing platform cannot be unknown, when pre_assembled is no.
        #   The correct answer here is Illumina.
        #   collection date should follow our format: 1998-05-29
        # Minor issues:
        #   City should be Melbourrne
        #   Region should be empty or Victoria
        #   Organism should only be the genus species annotation, anything else
        #   should be left out  (Enterococcus faecium)
        #   notes contain a lot of unnecessary spaces, and it does not contain
        #   the SRR identifier nor the biosample ID as would be preferred, and
        #   necessary for later annotation of the MLST to the data.
        match = re.findall(r'Run #1: (.+)\n', self.data)
        for answer in match:
            self.accession = answer.split(',')[0]

        match = re.findall(r'Sample Attributes: (.+)\n', self.data)
        for answer in match:
            for attributes in answer.split(';'):
                stat = attributes.split('=')
                att = stat[0].strip('/ ').lower().replace('\'', '')
                val = stat[1].strip('\' ').replace('\'', '\`')
                if att == 'geo_loc_name' and ':' in stat[1]:
                    self.metadata['country'] = val.split(':')[0]
                    self.metadata['region'] = val.split(':')[1]
                elif att == 'geo_loc_name':
                    self.metadata['country'] = val
                elif att == 'serovar':
                    self.metadata['subtype']['serovar'] = val
                elif att == 'mlst':
                    self.metadata['subtype']['mlst'] = val
{'fraction_non_alphanumeric': 0.12612612612612611, 'fraction_numerical': 0.01951951951951952, 'mean_word_length': 1.3411247803163444, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1029657', 'n_tokens_mistral': 4087, 'n_tokens_neox': 3705, 'n_words': 1131}