<commit_before>#!/usr/bin/env python
import simplejson
import json
import urllib
import time
import re
import BeautifulSoup
from BeautifulSoup import BeautifulStoneSoup
import nose
from nose.tools import assert_equals
from BasePlugin import BasePluginClass
from BasePlugin import TestBasePluginClass
import os

# Permissions: RWX for owner, WX for others.  Set this here so that .pyc are created with these permissions
os.umask(022) 
    
# Conforms to API specified here:  https://github.com/mhahnel/Total-Impact/wiki/Plugin-requirements
# To do automated tests with nosy                
# nosy CrossrefPlugin.py -A \'not skip\'
    
def skip(f):
    f.skip = True
    return f

class PluginClass(BasePluginClass):
                
    # each plugin needs to customize this stuff                
    SOURCE_NAME = "CrossRef"
    SOURCE_DESCRIPTION = "An official Digital Object Identifier (DOI) Registration Agency of the International DOI Foundation."
    SOURCE_URL = "http://www.crossref.org/"
    SOURCE_ICON = "http://www.crossref.org/favicon.ico"
    SOURCE_METRICS = dict(  journal="the journal where the paper was published",
                            year="the year of the publication",
                            title="the title of the publication", 
                            authors="the authors of the publication", 
                            doi="the DOI of the publication, if applicable",
                            url="the url of the full text of the publication",
                            pmid="the PubMed identifier of the publication, if applicable")

    DEBUG = False

    DOI_LOOKUP_URL = "http://dx.doi.org/%s"

    def get_page(self, doi):
        ## curl -D - -L -H "Accept: application/unixref+xml" "http://dx.doi.org/10.1126/science.1157784" 
        if not doi:
            return(None)
        url = self.DOI_LOOKUP_URL % doi
        if (self.DEBUG):
            print url
        try:
            page = self.get_cache_timeout_response(url, header_addons={'Accept':'application/unixref+xml'})
            if (self.DEBUG):
                print page
        except:
            page = None
        return(page)

    def extract_stats(self, page, doi):
        # crossref extraction code based on example at https://gist.github.com/931878
        if not page:
            return(None)        
        (response_header, content) = page
    
        soup = BeautifulStoneSoup(content)
        try:
            title = str(soup.title.text)
            if (title == "DOI Not Found"):
                return(None)
        except:
            title = ""

        try:
            year = str(soup.year.text)
        except:
            year = ""        
        
        try:
            journal = str(soup.abbrev_title.text)
        except:
            journal = ""
        
        try:
            authors = ", ".join([str(a.surname.text) for a in soup.findAll(contributor_role="author")])
        except:
            authors = ""
    
        # To get full text, try to follow the doi url then get the final landing page
        doi_initial_url = self.DOI_LOOKUP_URL % doi
        (redirected_header, redirected_page) = self.get_cache_timeout_response(doi_initial_url)
        try:
            url = redirected_header["content-location"]
        except:
            url = ""
       
        response = dict(url=url, title=title, journal=journal, year=year, authors=authors)
        return(response)  
    
    def get_metric_values(self, doi):
        page = self.get_page(doi)
        if page:
            response = self.extract_stats(page, doi)    
        else:
            response = None
        return(response)    
    
    def get_doi_from_pmid(self, pmid):
        url = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=%s&retmode=xml&email=%s" % (pmid, self.TOOL_EMAIL)
        (response, xml) = self.get_cache_timeout_response(url)
        try:
            doi = re.search('<ArticleId IdType="doi">(?P<doi>.*?)</ArticleId>', xml).group("doi")
        except:
            doi = ""
        return(doi)

    def get_pmid_from_doi(self, doi):
        url = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?term=%s&email=%s" % (doi, self.TOOL_EMAIL)
        (response, xml) = self.get_cache_timeout_response(url)
        try:
            pmid = re.search("<Id>(?P<pmid>\d*)</Id>", xml).group("pmid")
        except:
            pmid = ""
        return(pmid)
            
    def artifact_type_recognized(self, id):
        is_recognized = (self.is_crossref_doi(id) or self.is_pmid(id))
        return(is_recognized)   <commit_msg>exclude Dryad dois from what CrossRef plugin recognizes<commit_after>#!/usr/bin/env python
import simplejson
import json
import urllib
import time
import re
import BeautifulSoup
from BeautifulSoup import BeautifulStoneSoup
import nose
from nose.tools import assert_equals
from BasePlugin import BasePluginClass
from BasePlugin import TestBasePluginClass
import os

# Permissions: RWX for owner, WX for others.  Set this here so that .pyc are created with these permissions
os.umask(022) 
    
# Conforms to API specified here:  https://github.com/mhahnel/Total-Impact/wiki/Plugin-requirements
# To do automated tests with nosy                
# nosy CrossrefPlugin.py -A \'not skip\'
    
def skip(f):
    f.skip = True
    return f

class PluginClass(BasePluginClass):
                
    # each plugin needs to customize this stuff                
    SOURCE_NAME = "CrossRef"
    SOURCE_DESCRIPTION = "An official Digital Object Identifier (DOI) Registration Agency of the International DOI Foundation."
    SOURCE_URL = "http://www.crossref.org/"
    SOURCE_ICON = "http://www.crossref.org/favicon.ico"
    SOURCE_METRICS = dict(  journal="the journal where the paper was published",
                            year="the year of the publication",
                            title="the title of the publication", 
                            authors="the authors of the publication", 
                            doi="the DOI of the publication, if applicable",
                            url="the url of the full text of the publication",
                            pmid="the PubMed identifier of the publication, if applicable")

    DEBUG = False

    DOI_LOOKUP_URL = "http://dx.doi.org/%s"

    def get_page(self, doi):
        ## curl -D - -L -H "Accept: application/unixref+xml" "http://dx.doi.org/10.1126/science.1157784" 
        if not doi:
            return(None)
        url = self.DOI_LOOKUP_URL % doi
        if (self.DEBUG):
            print url
        try:
            page = self.get_cache_timeout_response(url, header_addons={'Accept':'application/unixref+xml'})
            if (self.DEBUG):
                print page
        except:
            page = None
        return(page)

    def extract_stats(self, page, doi):
        # crossref extraction code based on example at https://gist.github.com/931878
        if not page:
            return(None)        
        (response_header, content) = page
    
        soup = BeautifulStoneSoup(content)
        try:
            title = str(soup.title.text)
            if (title == "DOI Not Found"):
                return(None)
        except:
            title = ""

        try:
            year = str(soup.year.text)
        except:
            year = ""        
        
        try:
            journal = str(soup.abbrev_title.text)
        except:
            journal = ""
        
        try:
            authors = ", ".join([str(a.surname.text) for a in soup.findAll(contributor_role="author")])
        except:
            authors = ""
    
        # To get full text, try to follow the doi url then get the final landing page
        doi_initial_url = self.DOI_LOOKUP_URL % doi
        (redirected_header, redirected_page) = self.get_cache_timeout_response(doi_initial_url)
        try:
            url = redirected_header["content-location"]
        except:
            url = ""
       
        response = dict(url=url, title=title, journal=journal, year=year, authors=authors)
        return(response)  
    
    def get_metric_values(self, doi):
        page = self.get_page(doi)
        if page:
            response = self.extract_stats(page, doi)    
        else:
            response = None
        return(response)    
    
    def get_doi_from_pmid(self, pmid):
        url = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=%s&retmode=xml&email=%s" % (pmid, self.TOOL_EMAIL)
        (response, xml) = self.get_cache_timeout_response(url)
        try:
            doi = re.search('<ArticleId IdType="doi">(?P<doi>.*?)</ArticleId>', xml).group("doi")
        except:
            doi = ""
        return(doi)

    def get_pmid_from_doi(self, doi):
        url = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?term=%s&email=%s" % (doi, self.TOOL_EMAIL)
        (response, xml) = self.get_cache_timeout_response(url)
        try:
            pmid = re.search("<Id>(?P<pmid>\d*)</Id>", xml).group("pmid")
        except:
            pmid = ""
        return(pmid)
           
    def is_dryad_doi(self, id):        
        DRYAD_DOI_PATTERN = re.compile(r"10.5061/dryad", re.DOTALL)        
        response = (DRYAD_DOI_PATTERN.search(id) != None)
        return(response)
            
    def is_non_crossref_artifact(self, id):
        response = self.is_dryad_doi(id)
        return(response)
                
    def artifact_type_recognized(self, id):
        is_recognized = (self.is_crossref_doi(id) or self.is_pmid(id))
        if self.is_non_crossref_artifact(id):
            is_recognized = False
        return(is_recognized)   
{'fraction_non_alphanumeric': 0.10196280991735537, 'fraction_numerical': 0.005165289256198347, 'mean_word_length': 1.9962859795728876, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6313871', 'n_tokens_mistral': 3003, 'n_tokens_neox': 2822, 'n_words': 691}