<commit_before>import sys
import logging
import copy

from urllib2 import URLError
from httplib import HTTPException

from Bio import Entrez

RESULTS_FILE = 'dois-1000-per-line.txt'

LOG_FORMAT = '%(asctime)-15s %(message)s'
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
log = logging.getLogger(__name__)

def fail(msg, original_exception):
    global log
    log.critical(msg)
    raise original_exception

def to_file(filename, s):
    try:
        with open(filename, 'wb') as f:
            f.write(str(s))
    except IOError as e:
        fail(e, '''Could not open results file for writing.
IOError {}.
Data to write {}.'''.format(e, s))

def append_file(filename, s):
    try:
        with open(filename, 'ab') as f:
            f.write(str(s))
    except IOError as e:
        fail(e, '''Could not open results file for appending.
IOError {}.
Data to write {}.'''.format(e, s))


def main(argv=None):
    if not argv:
        argv = sys.argv

    resume_at_result_number = 0
    resuming = False
    if len(argv) > 1:
        if argv[1]:
            resume_at_result_number = argv[1]
            resuming = True
    
    Entrez.email = '<EMAIL>'
    
    query = "Wellcome[GRNT]"
    
    log.info('Sending this query to NCBI: {0}'.format(query))
    log.info('Starting from result number: {}'.format(resume_at_result_number))
   
    try:
        handle = Entrez.esearch(db="pubmed", term=query, retmax=100000,
                retstart=resume_at_result_number)
    except URLError as e:
        fail(e, '''NCBI query failed due to an URL Error. Are you connected
        to the internet? (It could be that the EUtils API is down or
        Biopython is generating the wrong URL.)''')

    record = Entrez.read(handle, validate=False)
    
    log.info('NCBI holds {} records related to this query.'.format(record['Count']))
    
    results = OAGPrep(RESULTS_FILE, resuming=resuming)
    
    for pmid in record['IdList']:
        try:
            individual_handle = Entrez.efetch(db='pubmed', retmode='xml',
                    id=pmid)
            individual_record = Entrez.read(individual_handle,
                    validate=False)
        except ValueError as e:
            log.warn('''ValueError, Biopython probably couldn\'t parse
            something or the returned XML was invalid. Skipping PMID {}.
            Original error {}'''.format(pmid, e))
            continue
        except (URLError, HTTPException) as e:
            log.warn('''Networking error. Skipping PMID {}.
            Original error {}'''.format(pmid, e))
            continue
    
        if len(individual_record) > 1:
            log.warn('PMID {}: NCBI response contains multiple items in the individual record list'.format(pmid))
    
        if 'PubmedData' not in individual_record[0]:
            log.warn('PMID {}: NCBI response did not contain PubmedData key'.format(pmid))
    
        if 'ArticleIdList' not in individual_record[0]['PubmedData']:
            log.warn('PMID {}: NCBI response did not contain the ArticleIdList key in the PubmedData dict'.format(pmid))
    
        for identifier in individual_record[0]['PubmedData']['ArticleIdList']:
            try:
                if identifier.attributes['IdType'] == 'doi':
                    results.add(identifier)
            except AttributeError:
                log.warn('PMID {}: Can\'t add PMID, no associated DOI.'.format(pmid))

class OAGPrep:
    current_row = []
    rows = [current_row]<commit_msg>make lists of skipped and problematic PMIDs as well<commit_after>import sys
import logging
import copy

from urllib2 import URLError
from httplib import HTTPException

from Bio import Entrez

RESULTS_FILE = 'dois-1000-per-line.txt'

LOG_FORMAT = '%(asctime)-15s %(message)s'
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
log = logging.getLogger(__name__)

def fail(msg, original_exception):
    global log
    log.critical(msg)
    raise original_exception

def warn_skip(msg, pmid):
    global log
    log.warn(msg)
    append_file('skipped_pmids.txt', ',' + pmid)

def warn_problem_ncbi_record(msg, pmid):
    global log
    log.warn(msg)
    append_file('problem_pmids.txt', ',' + pmid)

def to_file(filename, s):
    try:
        with open(filename, 'wb') as f:
            f.write(str(s))
    except IOError as e:
        fail(e, '''Could not open results file for writing.
IOError {}.
Data to write {}.'''.format(e, s))

def append_file(filename, s):
    try:
        with open(filename, 'ab') as f:
            f.write(str(s))
    except IOError as e:
        fail(e, '''Could not open results file for appending.
IOError {}.
Data to write {}.'''.format(e, s))

def main(argv=None):
    if not argv:
        argv = sys.argv

    resume_at_result_number = 0
    resuming = False
    if len(argv) > 1:
        if argv[1]:
            resume_at_result_number = argv[1]
            resuming = True
    
    Entrez.email = '<EMAIL>'
    
    query = "Wellcome[GRNT]"
    
    log.info('Sending this query to NCBI: {0}'.format(query))
    log.info('Starting from result number: {}'.format(resume_at_result_number))
   
    try:
        handle = Entrez.esearch(db="pubmed", term=query, retmax=100000,
                retstart=resume_at_result_number)
    except URLError as e:
        fail(e, '''NCBI query failed due to an URL Error. Are you connected
        to the internet? (It could be that the EUtils API is down or
        Biopython is generating the wrong URL.)''')

    record = Entrez.read(handle, validate=False)
    
    log.info('NCBI holds {} records related to this query.'.format(record['Count']))
    
    results = OAGPrep(RESULTS_FILE, resuming=resuming)
    
    for pmid in record['IdList']:
        try:
            individual_handle = Entrez.efetch(db='pubmed', retmode='xml',
                    id=pmid)
            individual_record = Entrez.read(individual_handle,
                    validate=False)
        except ValueError as e:
            warn_skip(
'''ValueError, Biopython probably couldn\'t parse
something or the returned XML was invalid. Skipping PMID {}.
Original error {}'''.format(pmid, e),
                pmid)
            continue
        except (URLError, HTTPException) as e:
            warn_skip(
'''Networking error. Skipping PMID {}.
Original error {}'''.format(pmid, e),
                pmid)
            continue
    
        if len(individual_record) > 1:
            warn_problem_ncbi_record('PMID {}: NCBI response contains multiple items in the individual record list'.format(pmid), pmid)
    
        if 'PubmedData' not in individual_record[0]:
            warn_problem_ncbi_record('PMID {}: NCBI response did not contain PubmedData key'.format(pmid), pmid)
    
        if 'ArticleIdList' not in individual_record[0]['PubmedData']:
            warn_problem_ncbi_record('PMID {}: NCBI response did not contain the ArticleIdList key in the PubmedData dict'.format(pmid), pmid)
    
        for identifier in individual_record[0]['PubmedData']['ArticleIdList']:
            try:
                if identifier.attributes['IdType'] == 'doi':
                    results.add(identifier)
            except AttributeError:
                warn_problem_ncbi_record('PMID {}: Can\'t add PMID, no associated DOI.'.format(pmid), pmid)

class OAGPrep:
    current_row = []
    rows = [current_row]
{'fraction_non_alphanumeric': 0.11080860332153553, 'fraction_numerical': 0.005989654233596515, 'mean_word_length': 2.576923076923077, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1656167', 'n_tokens_mistral': 2296, 'n_tokens_neox': 2194, 'n_words': 625}