<commit_before>"""Create publication class and contain methods for data fetching."""
import requests
import json
from snovault import (
    collection,
    load_schema,
    calculated_property
)
from snovault.attachment import ItemWithAttachment
from .base import (
    Item,
    lab_award_attribution_embed_list
)
from html.parser import HTMLParser
from encoded.types.experiment_set import invalidate_linked_items

################################################
# Outside methods for online data fetch
################################################


def find_best_date(date_data):
    date = None
    a2d = {'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06',
           'Jul': '07', 'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'}
    if 'DP' in date_data:
        ymd = [d.strip() for d in date_data['DP'].split(' ')]
        if not ymd or len(ymd[0]) != 4:  # problem with the year
            pass
        else:
            date = ymd.pop(0)
            if ymd:
                mm = a2d.get(ymd.pop(0))
                if mm:
                    date += '-{}'.format(mm)
                    if ymd:
                        dd = ymd.pop(0)
                        if len(dd) <= 2:
                            date += '-{}'.format(dd.zfill(2))
            return date
    if 'DEP' in date_data:
        date = date_data['DEP']
        if len(date) != 8:
            date = None
    if not date and 'DA' in date_data:
        date = date_data['DA']
    if date:
        datestr = date[:4]+"-"+date[4:6]+"-"+date[6:8]
        if len(datestr) == 10:
            return datestr
    return None


def fetch_pubmed(PMID):
    "Takes the number part of PMID and returns title, abstract and authors"

    field2prop = {'TI': 'title', 'AB': 'abstract', 'DP': 'date_published',
                  'DEP': 'date_published', 'DA': 'date_published', 'JT': 'journal',
                  'AU': 'authors', 'CN': 'authors'}
    pub_data = {v: None for v in field2prop.values()}
    pub_data['authors'] = []
    pub_data['date_published'] = {}
    NIHe = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
    NIHw = "https://www.ncbi.nlm.nih.gov/pubmed/"
    url = NIHw + PMID
    pub_data['url'] = url
    www = "{NIH}efetch.fcgi?db=pubmed&id={id}&rettype=medline".format(NIH=NIHe, id=PMID)
    # try fetching data 5 times
    for count in range(5):
        resp = requests.get(www)
        if resp.status_code == 200:
            break
        if count == 4:
            return {}
    # parse the text to get the fields
    r = resp.text
    full_text = r.replace('\n      ', ' ')
    for line in full_text.split('\n'):
        if line.strip():
            key, val = [a.strip() for a in line.split('-', 1)]
            if key in field2prop:
                if key in ['DP', 'DEP', 'DA']:
                    pub_data[field2prop[key]][key] = val
                elif key in ['AU', 'CN']:
                    pub_data[field2prop[key]].append(val)
                else:
                    pub_data[field2prop[key]] = val
    # deal with date
    if pub_data['date_published']:  # there is some date data
        pub_data['date_published'] = find_best_date(pub_data['date_published'])
    return {k: v for k, v in pub_data.items() if v is not None}
    return pub_data


class BioRxivExtractor(HTMLParser):
    def __init__(self):
        HTMLParser.__init__(self)
        self.title = ''
        self.abstract = ''
        self.authors = []
        self.date_published = ''

    def handle_starttag(self, tag, attrs):
        attr = ''
        if tag == 'meta':
            attr = {k[0]: k[1] for k in attrs}
            if attr.get('name') == "DC.Title":
                self.title = attr.get('content')
            if attr.get('name') == "DC.Description":
                self.abstract = attr.get('content')
            if attr.get('name') == "DC.Contributor":
                self.authors.append(attr.get('content'))
            if attr.get('name') == "DC.Date":
                self.date_published = attr.get('content')


def fetch_biorxiv(url):<commit_msg>codacy tweak<commit_after>"""Create publication class and contain methods for data fetching."""
import requests
import json
from snovault import (
    collection,
    load_schema,
    calculated_property
)
from snovault.attachment import ItemWithAttachment
from .base import (
    Item,
    lab_award_attribution_embed_list
)
from html.parser import HTMLParser
from encoded.types.experiment_set import invalidate_linked_items

################################################
# Outside methods for online data fetch
################################################


def find_best_date(date_data):
    date = None
    a2d = {'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06',
           'Jul': '07', 'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'}
    if 'DP' in date_data:
        ymd = [d.strip() for d in date_data['DP'].split(' ')]
        if not ymd or len(ymd[0]) != 4:  # problem with the year
            pass
        else:
            date = ymd.pop(0)
            if ymd:
                mm = a2d.get(ymd.pop(0))
                if mm:
                    date += '-{}'.format(mm)
                    if ymd:
                        dd = ymd.pop(0)
                        if len(dd) <= 2:
                            date += '-{}'.format(dd.zfill(2))
            return date
    if 'DEP' in date_data:
        date = date_data['DEP']
        if len(date) != 8:
            date = None
    if not date and 'DA' in date_data:
        date = date_data['DA']
    if date:
        datestr = date[:4]+"-"+date[4:6]+"-"+date[6:8]
        if len(datestr) == 10:
            return datestr
    return None


def fetch_pubmed(PMID):
    "Takes the number part of PMID and returns title, abstract and authors"

    field2prop = {'TI': 'title', 'AB': 'abstract', 'DP': 'date_published',
                  'DEP': 'date_published', 'DA': 'date_published', 'JT': 'journal',
                  'AU': 'authors', 'CN': 'authors'}
    pub_data = {v: None for v in field2prop.values()}
    pub_data['authors'] = []
    pub_data['date_published'] = {}
    NIHe = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
    NIHw = "https://www.ncbi.nlm.nih.gov/pubmed/"
    url = NIHw + PMID
    pub_data['url'] = url
    www = "{NIH}efetch.fcgi?db=pubmed&id={id}&rettype=medline".format(NIH=NIHe, id=PMID)
    # try fetching data 5 times
    for count in range(5):
        resp = requests.get(www)
        if resp.status_code == 200:
            break
        if count == 4:
            return {}
    # parse the text to get the fields
    r = resp.text
    full_text = r.replace('\n      ', ' ')
    for line in full_text.split('\n'):
        if line.strip():
            key, val = [a.strip() for a in line.split('-', 1)]
            if key in field2prop:
                if key in ['DP', 'DEP', 'DA']:
                    pub_data[field2prop[key]][key] = val
                elif key in ['AU', 'CN']:
                    pub_data[field2prop[key]].append(val)
                else:
                    pub_data[field2prop[key]] = val
    # deal with date
    if pub_data['date_published']:  # there is some date data
        pub_data['date_published'] = find_best_date(pub_data['date_published'])
    return {k: v for k, v in pub_data.items() if v is not None}


class BioRxivExtractor(HTMLParser):
    def __init__(self):
        HTMLParser.__init__(self)
        self.title = ''
        self.abstract = ''
        self.authors = []
        self.date_published = ''

    def handle_starttag(self, tag, attrs):
        attr = ''
        if tag == 'meta':
            attr = {k[0]: k[1] for k in attrs}
            if attr.get('name') == "DC.Title":
                self.title = attr.get('content')
            if attr.get('name') == "DC.Description":
                self.abstract = attr.get('content')
            if attr.get('name') == "DC.Contributor":
                self.authors.append(attr.get('content'))
            if attr.get('name') == "DC.Date":
                self.date_published = attr.get('content')


def fetch_biorxiv(url):
{'fraction_non_alphanumeric': 0.16909692439652002, 'fraction_numerical': 0.013723808356819018, 'mean_word_length': 2.047796863330844, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2865281', 'n_tokens_mistral': 2680, 'n_tokens_neox': 2542, 'n_words': 666}