<commit_before>from totalimpact.providers import provider
from totalimpact.providers.provider import Provider, ProviderContentMalformedError

import simplejson, re, os, random, string

import logging
logger = logging.getLogger('ti.providers.scopus')

class Scopus(Provider):  

    example_id = ("doi", "10.1371/journal.pone.0000308")

    url = "http://www.info.sciverse.com/scopus/about"
    descr = "The world's largest abstract and citation database of peer-reviewed literature."
    # template urls below because they need a freshly-minted random string
    metrics_url_template = None
    provenance_url_template = None

    static_meta_dict =  { 
        "citations": {
            "display_name": "citations",
            "provider": "Scopus",
            "provider_url": "http://www.info.sciverse.com/scopus/about",
            "description": "Number of times the item has been cited",
            "icon": "http://www.info.sciverse.com/sites/all/themes/sciverse/favicon.ico" ,
        }
    }
    

    def __init__(self):
        super(Scopus, self).__init__()

    def is_relevant_alias(self, alias):
        (namespace, nid) = alias
        return("doi" == namespace)


    def _get_json(self, fullpage, id):
        try:
            # extract json from inside the first and last parens
            # from http://codereview.stackexchange.com/questions/2561/converting-jsonp-to-json-is-this-regex-correct
            page = fullpage[ fullpage.index("(")+1 : fullpage.rindex(")") ]
        except ValueError:
            raise ProviderContentMalformedError()

        data = provider._load_json(page)
        return(data)

    def _get_relevant_record(self, fullpage, id):
        data = self._get_json(fullpage, id)
        response = None
        try:
            citation_rows = data["OK"]["results"]
            for citation_row in citation_rows:
                if citation_row["doi"]==id:
                    response = citation_row
        except (KeyError, ValueError):
            # not in Scopus database
            return None
        return response

    def _extract_metrics(self, fullpage, status_code=200, id=None):
        record = self._get_relevant_record(fullpage, id)
        try:
            citations = int(record["citedbycount"])    
        except (KeyError, TypeError):
            return {}
        if citations:
            metrics_dict = {"scopus:citations": citations}
        return metrics_dict

    def _extract_provenance_url(self, fullpage, status_code=200, id=None):
        record = self._get_relevant_record(fullpage, id)
        try:
            provenance_url = record["inwardurl"] 
        except (KeyError, TypeError):
            provenance_url = ""
        return provenance_url

    def _get_page(self, url):
        response = self.http_get(url, timeout=30)
        if response.status_code != 200:
            if response.status_code == 404:
                return None
            else:
                raise(self._get_error(response.status_code))
        page = response.text
        if not page:
            raise ProviderContentMalformedError()
        return page

    def _get_page_with_doi(self, provider_url_template, id):
        # pick a new random string so don't time out.  Unfort, url now can't cache.
        random_string = "".join(random.sample(string.letters, 10))
        self.metrics_url_template = 'http://searchapi.scopus.com/documentSearch.url?&search=%s&callback=sciverse.Backend._requests.search1.callback&preventCache='+random_string+"&apiKey="+os.environ["SCOPUS_KEY"]
        self.provenance_url_template = self.metrics_url_template

        if not provider_url_template:
            provider_url_template = self.metrics_url_template

        logger.debug("id = {id}".format(id=id))
        logger.debug("provider_url_template = {provider_url_template}".format(
            provider_url_template=provider_url_template))

        url = self._get_templated_url(provider_url_template, id, "metrics")
        print url
        page = self._get_page(url)
        relevant_record = self._get_relevant_record(page, id)
        if not relevant_record:
            data = self._get_json(page, id)
            try:
                number_results = data["OK"]["totalResults"]
            except (KeyError, ValueError):
                return None            
            url = "{previous_url}&offset={last_record}".format(
                previous_url=url, last_record=(int(number_results)-1))
            print url
            page = self._get_page(url)
            relevant_record = self._get_relevant_record(page, id)
            if not relevant_record:
                return None
        return page

    def _get_metrics_and_drilldown_from_metrics_page(self, provider_url_template, id):
        page = self._get_page_with_doi(provider_url_template, id)
        if not page:
            logging.warning("couldn't find a page with doi {id}".format(id=id))<commit_msg>no error for no results in page<commit_after>from totalimpact.providers import provider
from totalimpact.providers.provider import Provider, ProviderContentMalformedError

import simplejson, re, os, random, string

import logging
logger = logging.getLogger('ti.providers.scopus')

class Scopus(Provider):  

    example_id = ("doi", "10.1371/journal.pone.0000308")

    url = "http://www.info.sciverse.com/scopus/about"
    descr = "The world's largest abstract and citation database of peer-reviewed literature."
    # template urls below because they need a freshly-minted random string
    metrics_url_template = None
    provenance_url_template = None

    static_meta_dict =  { 
        "citations": {
            "display_name": "citations",
            "provider": "Scopus",
            "provider_url": "http://www.info.sciverse.com/scopus/about",
            "description": "Number of times the item has been cited",
            "icon": "http://www.info.sciverse.com/sites/all/themes/sciverse/favicon.ico" ,
        }
    }
    

    def __init__(self):
        super(Scopus, self).__init__()

    def is_relevant_alias(self, alias):
        (namespace, nid) = alias
        return("doi" == namespace)


    def _get_json(self, fullpage, id):
        try:
            # extract json from inside the first and last parens
            # from http://codereview.stackexchange.com/questions/2561/converting-jsonp-to-json-is-this-regex-correct
            page = fullpage[ fullpage.index("(")+1 : fullpage.rindex(")") ]
        except ValueError:
            raise ProviderContentMalformedError()

        data = provider._load_json(page)
        return(data)

    def _get_relevant_record(self, fullpage, id):
        data = self._get_json(fullpage, id)
        response = None
        try:
            citation_rows = data["OK"]["results"]
            for citation_row in citation_rows:
                if citation_row["doi"]==id:
                    response = citation_row
        except (KeyError, ValueError):
            # not in Scopus database
            return None
        return response

    def _extract_metrics(self, fullpage, status_code=200, id=None):
        record = self._get_relevant_record(fullpage, id)
        try:
            citations = int(record["citedbycount"])    
        except (KeyError, TypeError):
            return {}
        if citations:
            metrics_dict = {"scopus:citations": citations}
        return metrics_dict

    def _extract_provenance_url(self, fullpage, status_code=200, id=None):
        record = self._get_relevant_record(fullpage, id)
        try:
            provenance_url = record["inwardurl"] 
        except (KeyError, TypeError):
            provenance_url = ""
        return provenance_url

    def _get_page(self, url):
        response = self.http_get(url, timeout=30)
        if response.status_code != 200:
            if response.status_code == 404:
                return None
            else:
                raise(self._get_error(response.status_code))
        page = response.text
        if not page:
            raise ProviderContentMalformedError()
        return page

    def _get_page_with_doi(self, provider_url_template, id):
        # pick a new random string so don't time out.  Unfort, url now can't cache.
        random_string = "".join(random.sample(string.letters, 10))
        self.metrics_url_template = 'http://searchapi.scopus.com/documentSearch.url?&search=%s&callback=sciverse.Backend._requests.search1.callback&preventCache='+random_string+"&apiKey="+os.environ["SCOPUS_KEY"]
        self.provenance_url_template = self.metrics_url_template

        if not provider_url_template:
            provider_url_template = self.metrics_url_template

        logger.debug("id = {id}".format(id=id))
        logger.debug("provider_url_template = {provider_url_template}".format(
            provider_url_template=provider_url_template))

        url = self._get_templated_url(provider_url_template, id, "metrics")
        page = self._get_page(url)
        if "Result set was empty" in page:
            return None
        relevant_record = self._get_relevant_record(page, id)
        if not relevant_record:
            data = self._get_json(page, id)
            try:
                number_results = data["OK"]["totalResults"]
            except (KeyError, ValueError):
                return None            
            url = "{previous_url}&offset={last_record}".format(
                previous_url=url, last_record=(int(number_results)-1))
            page = self._get_page(url)
            relevant_record = self._get_relevant_record(page, id)
            if not relevant_record:
                logging.warning("not empty result set, yet couldn't find a page with doi {id}".format(id=id))
                return None
        return page

    def _get_metrics_and_drilldown_from_metrics_page(self, provider_url_template, id):
        page = self._get_page_with_doi(provider_url_template, id)
        if not page:
            logging.info("no scopus page with doi {id}".format(id=id))
{'fraction_non_alphanumeric': 0.09878389154704945, 'fraction_numerical': 0.007177033492822967, 'mean_word_length': 2.4656303972366147, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1533450', 'n_tokens_mistral': 3001, 'n_tokens_neox': 2813, 'n_words': 619}