<commit_before>#!/usr/bin/env python3
"""
Created on Fri Apr 28 11:30:56 2017
@author: <NAME>

@Dependencies: Requires 'PubMedLookup' Python module from https://github.com/mfcovington/pubmed-lookup
"""

from pubmed_lookup import PubMedLookup
from pubmed_lookup import Publication
import pandas as pd
import os

def search(PubMedID):
	# Retrieve a PubMed record:
	# NCBI will contact user by email if excessive queries are detected
	"""
	Retrieve a PubMed record using its PubMed ID or PubMed URL.
	(e.g., '22331878' or 'http://www.ncbi.nlm.nih.gov/pubmed/22331878')
	"""
	email = ''
	#url = 'http://www.ncbi.nlm.nih.gov/pubmed/22331878'
	#lookup = PubMedLookup(url, email)
	#print(type(PubMedID))
	lookup = PubMedLookup(PubMedID,email)

	# Create a Publication object:
	publication = Publication(lookup)    # Use 'resolve_doi=False' to keep DOI URL

	# Access the Publication object's attributes:
	tupleoutput = [(
		PubMedID,
		publication.title,
		publication.authors,
		publication.journal,
		publication.year,
		publication.month,
		publication.day,
		publication.url,
		publication.pubmed_url,
		publication.cite(),
		publication.cite_mini(),
		repr(publication.abstract),
		0,
	)]
	
	return tupleoutput
	
#print(search('','<EMAIL>'))
#print(search(22331878))

for id_no in range(10000001,28470030):
	#print(tuple(id_no))
	df = pd.DataFrame(search(id_no), columns=['PubMedID','title','authors','journal','year','month','day','url','pubmed','citation','mini_citation','abstract','pagerank'])

	# if file does not exist write header 
	if not os.path.isfile('pubmed_index.csv'):
		print('Parsing id_no #',id_no,'...'),
		df.to_csv('pubmed_index.csv',header ='column_names')
	elif pd.read_csv('pubmed_index.csv')['PubMedID'].max() >= id_no:
		print('repeat no:',id_no)
	else: # else it exists so append without writing the header
		print('Parsing id_no #',id_no,'...'),
		df.to_csv('pubmed_index.csv',mode = 'a',header=False)


<commit_msg>modified 'elif' line, such that it checks if any entries of specified id no. exist<commit_after>#!/usr/bin/env python3
"""
Created on Fri Apr 28 11:30:56 2017
@author: <NAME>

@Dependencies: Requires 'PubMedLookup' Python module from https://github.com/mfcovington/pubmed-lookup
"""

from pubmed_lookup import PubMedLookup
from pubmed_lookup import Publication
import pandas as pd
import os

def search(PubMedID):
	# Retrieve a PubMed record:
	# NCBI will contact user by email if excessive queries are detected
	"""
	Retrieve a PubMed record using its PubMed ID or PubMed URL.
	(e.g., '22331878' or 'http://www.ncbi.nlm.nih.gov/pubmed/22331878')
	"""
	email = ''
	#url = 'http://www.ncbi.nlm.nih.gov/pubmed/22331878'
	#lookup = PubMedLookup(url, email)
	#print(type(PubMedID))
	lookup = PubMedLookup(PubMedID,email)

	# Create a Publication object:
	publication = Publication(lookup)    # Use 'resolve_doi=False' to keep DOI URL

	# Access the Publication object's attributes:
	tupleoutput = [(
		PubMedID,
		publication.title,
		publication.authors,
		publication.journal,
		publication.year,
		publication.month,
		publication.day,
		#publication.url,	# doesn't seem to work w/ dataframe
		publication.pubmed_url,
		publication.cite(),
		publication.cite_mini(),
		repr(publication.abstract),
		0,
	)]
	
	return tupleoutput
	
#print(search('','<EMAIL>'))
#print(search(22331878))

for id_no in range(10019070, 28470030):
	#print(tuple(id_no))
#	df = pd.DataFrame(search(id_no), columns=['PubMedID','title','authors','journal','year','month','day','url','pubmed_url','citation','mini_citation','abstract','pagerank'])
	df = pd.DataFrame(search(id_no), columns=['PubMedID','title','authors','journal','year','month','day','pubmed_url','citation','mini_citation','abstract','pagerank'])

	# if file does not exist write header 
	if not os.path.isfile('pubmed_index.csv'):
		print('Parsing id_no #',id_no,'...'),
		df.to_csv('pubmed_index.csv',header ='column_names')
	elif any(pd.read_csv('pubmed_index.csv')['PubMedID']==id_no):
		print('repeat no:',id_no)
	else: # else it doesn't exists, so append without writing the header
		print('Parsing id_no #',id_no,'...'),
		df.to_csv('pubmed_index.csv',mode = 'a',header=False)



{'fraction_non_alphanumeric': 0.154307655387569, 'fraction_numerical': 0.029757619390448763, 'mean_word_length': 5.5023400936037445, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1202174', 'n_tokens_mistral': 1619, 'n_tokens_neox': 1421, 'n_words': 326}