<commit_before>from totalimpact.providers import provider
from totalimpact.providers.provider import Provider, ProviderContentMalformedError

import simplejson, re, os, random, string, urllib

import logging
logger = logging.getLogger('ti.providers.scopus')

class Scopus(Provider):  

    example_id = ("doi", "10.1371/journal.pone.0000308")

    url = "http://www.info.sciverse.com/scopus/about"
    descr = "The world's largest abstract and citation database of peer-reviewed literature."
    # template urls below because they need a freshly-minted random string
    metrics_url_template = None
    provenance_url_template = None

    static_meta_dict =  { 
        "citations": {
            "display_name": "citations",
            "provider": "Scopus",
            "provider_url": "http://www.info.sciverse.com/scopus/about",
            "description": "Number of times the item has been cited",
            "icon": "http://www.info.sciverse.com/sites/all/themes/sciverse/favicon.ico" ,
        }
    }
    

    def __init__(self):
        super(Scopus, self).__init__()

    def is_relevant_alias(self, alias):
        (namespace, nid) = alias
        return("doi" == namespace)


    def _get_json(self, fullpage):
        try:
            # extract json from inside the first and last parens
            # from http://codereview.stackexchange.com/questions/2561/converting-jsonp-to-json-is-this-regex-correct
            page = fullpage[ fullpage.index("(")+1 : fullpage.rindex(")") ]
        except (AttributeError, ValueError):
            raise ProviderContentMalformedError()

        data = provider._load_json(page)
        return(data)



    def _extract_metrics(self, record, status_code=200, id=None):
        try:
            citations = int(record["citedbycount"])    
        except (KeyError, TypeError):
            return {}

        if citations:
            metrics_dict = {"scopus:citations": citations}
        else:
            metrics_dict = {}                    
        return metrics_dict

    def _extract_provenance_url(self, record, status_code=200, id=None):
        try:
            provenance_url = record["inwardurl"] 
        except (KeyError, TypeError):
            provenance_url = ""
        return provenance_url

    def _get_page(self, url):
        response = self.http_get(url, timeout=30)
        if response.status_code != 200:
            if response.status_code == 404:
                return None<commit_msg>Catch ValueErrors for when Scopus provider tries to get citedByCount and gets an empty string<commit_after>from totalimpact.providers import provider
from totalimpact.providers.provider import Provider, ProviderContentMalformedError

import simplejson, re, os, random, string, urllib

import logging
logger = logging.getLogger('ti.providers.scopus')

class Scopus(Provider):  

    example_id = ("doi", "10.1371/journal.pone.0000308")

    url = "http://www.info.sciverse.com/scopus/about"
    descr = "The world's largest abstract and citation database of peer-reviewed literature."
    # template urls below because they need a freshly-minted random string
    metrics_url_template = None
    provenance_url_template = None

    static_meta_dict =  { 
        "citations": {
            "display_name": "citations",
            "provider": "Scopus",
            "provider_url": "http://www.info.sciverse.com/scopus/about",
            "description": "Number of times the item has been cited",
            "icon": "http://www.info.sciverse.com/sites/all/themes/sciverse/favicon.ico" ,
        }
    }
    

    def __init__(self):
        super(Scopus, self).__init__()

    def is_relevant_alias(self, alias):
        (namespace, nid) = alias
        return("doi" == namespace)


    def _get_json(self, fullpage):
        try:
            # extract json from inside the first and last parens
            # from http://codereview.stackexchange.com/questions/2561/converting-jsonp-to-json-is-this-regex-correct
            page = fullpage[ fullpage.index("(")+1 : fullpage.rindex(")") ]
        except (AttributeError, ValueError):
            raise ProviderContentMalformedError()

        data = provider._load_json(page)
        return(data)



    def _extract_metrics(self, record, status_code=200, id=None):
        try:
            citations = int(record["citedbycount"])    
        except (KeyError, TypeError, ValueError):
            return {}

        if citations:
            metrics_dict = {"scopus:citations": citations}
        else:
            metrics_dict = {}                    
        return metrics_dict

    def _extract_provenance_url(self, record, status_code=200, id=None):
        try:
            provenance_url = record["inwardurl"] 
        except (KeyError, TypeError):
            provenance_url = ""
        return provenance_url

    def _get_page(self, url):
        response = self.http_get(url, timeout=30)
        if response.status_code != 200:
            if response.status_code == 404:
                return None
{'fraction_non_alphanumeric': 0.10405889375248707, 'fraction_numerical': 0.012733784321528054, 'mean_word_length': 2.471685082872928, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5754480', 'n_tokens_mistral': 1541, 'n_tokens_neox': 1404, 'n_words': 332}