<commit_before>from . import ns, get_encoded_text, MY_API_KEY
import requests
import xml.etree.ElementTree as ET
import os
import sys

SCOPUS_XML_DIR = os.path.expanduser('~/.scopus/xml')
SCOPUS_ISSN_DIR = os.path.expanduser('~/.scopus/issn')

if not os.path.exists(SCOPUS_XML_DIR):
    os.makedirs(SCOPUS_XML_DIR)

if not os.path.exists(SCOPUS_ISSN_DIR):
    os.makedirs(SCOPUS_ISSN_DIR)


class ScopusAbstract(object):
    """Class to represent the results from a Scopus abstract.

    The results are retrieved by the EID from a query. The results
    are cached in a folder ~/.scopus/xml/{eid}.
    """

    @property
    def url(self):
        """URL to the abstract."""
        return self._url

    @property
    def doi(self):
        """Abstract DOI."""
        return self._doi

    @property
    def title(self):
        """Abstract title."""
        return self._title

    @property
    def aggregationType(self):
        """Type of abstract."""
        return self._aggregationType

    @property
    def publicationName(self):
        """Name of source the abstract is published in."""
        return self._publicationName

    @property
    def citedby_count(self):
        """Number of times the abstract has been cited."""
        return int(self._citedby_count)

    @property
    def publisher(self):
        """Name of the publisher of the abstract."""
        return self._publisher

    @property
    def source_id(self):
        """Scopus source_id of the abstract."""
        return self._source_id

    @property
    def issn(self):
        """ISSN of the publisher."""
        return self._issn

    @property
    def volume(self):
        """Volume for the abstract."""
        return self._volume

    @property
    def issueIdentifier(self):
        """Issue number for abstract."""
        return self._issueIdentifier

    @property
    def article_number(self):
        """Article number."""
        return self._article_number

    @property
    def startingPage(self):
        """Starting page."""
        return self._startingPage

    @property
    def endingPage(self):
        """Ending page."""
        return self._endingPage

    @property
    def pageRange(self):
        """Page range."""
        return self._pageRange

    @property
    def coverDate(self):
        """The date of the cover the abstract is in."""
        return self._coverDate

    @property
    def authors(self):
        """A list of scopus_api.ScopusAuthor objects."""
        return self._authors

    @property
    def affiliations(self):
        """A list of scopus_api.ScopusAffiliation objects."""
        return self._affiliations

    @property
    def nauthors(self):
        """Return number of authors listed in the abstract."""
        return len(self.authors)

    @property
    def refcount(self):
        """Return number of references of an article, if possible."""
        if self._references is not None:
            return self._references.attrib['refcount']
        else:
            raise TypeError("Could not load article references. "
                            "Did you load with view=FULL?")

    @property
    def references(self):
        """Return EIDs of references of an article."""
        if self._references is not None:
            eids = [r.find("ref-info/refd-itemidlist/itemid", ns).text for r
                    in self._references.findall("reference", ns)]
            return ["2-s2.0-" + eid for eid in eids]
        else:
            raise TypeError("Could not load article references. "
                            "Did you load with view=FULL?")

    def __init__(self, EID, view='META_ABS', refresh=False):
        """Initialize a Scopus abstract

        EID -- The scopus id for an abstract and should be a string.

        view -- (default=META_ABS) The view of the file that should be
        downloaded.  Currently supported values: META, META_ABS, FULL.

        refresh -- (default=False) A boolean that determines if the result
        should be downloaded again.
        """
        allowed_views = ('META', 'META_ABS', 'FULL')
        if view not in allowed_views:
            raise ValueError('view parameter must be one of ' +\
                             ', '.join(allowed_views))

        fEID = os.path.join(SCOPUS_XML_DIR, EID)
        self.file = fEID

        if os.path.exists(fEID) and not refresh:
            with open(fEID) as f:
                text = f.read()
                self.xml = text
                results = ET.fromstring(text)
        else:
            url = "http://api.elsevier.com/content/abstract/eid/{}".format(EID)

            resp = requests.get(url,
                                headers={'Accept': 'application/xml',
                                         'X-ELS-APIKey': MY_API_KEY},
                                params={'view': view})
            self.xml = resp.text
            with open(fEID, 'w') as f:
                if sys.version_info[0] == 3:
                    f.write(self.xml)
                else:
                    f.write(self.xml.encode('utf-8'))

            results = ET.fromstring(resp.text.encode('utf-8'))

        coredata = results.find('dtd:coredata', ns)
        authors = results.find('dtd:authors', ns)
        items = results.find('item', ns)
        self.results = results
        if results.tag == 'service-error':
            raise Exception('\n{0}\n{1}'.format(EID, self.xml))

        self.coredata = coredata
        self.authors_xml = authors
        self._url = get_encoded_text(coredata, 'prism:url')

        self.identifier = get_encoded_text(coredata, 'dc:identifier')
        self.eid = get_encoded_text(coredata, 'dtd:eid')
        self._doi = get_encoded_text(coredata, 'prism:doi')
        self._title = get_encoded_text(coredata, 'dc:title')

        self._aggregationType = get_encoded_text(coredata,
                                                 'prism:aggregationType')
        self._publicationName = get_encoded_text(coredata,
                                                 'prism:publicationName')
        self.srctype = get_encoded_text(coredata, 'dtd:srctype')
        self._citedby_count = get_encoded_text(coredata, 'dtd:citedby-count')
        self._publisher = get_encoded_text(coredata, 'dc:publisher')
        self._source_id = get_encoded_text(coredata, 'dtd:source-id')
        self._issn = get_encoded_text(coredata, 'prism:issn')
        self._volume = get_encoded_text(coredata, 'prism:volume')
        self._issueIdentifier = get_encoded_text(coredata,
                                                 'prism:issueIdentifier')
        self._article_number = get_encoded_text(coredata, 'dtd:article-number')
        self._startingPage = get_encoded_text(coredata, 'prism:startingPage')
        self._endingPage = get_encoded_text(coredata, 'prism:endingPage')
        self._pageRange = get_encoded_text(coredata, 'prism:pageRange')
        self._coverDate = get_encoded_text(coredata, 'prism:coverDate')
        self.creator = get_encoded_text(coredata, 'dc:creator')
        self.description = get_encoded_text(coredata, 'dc:description')

        sl = coredata.find('dtd:link[@rel="scopus"]', ns).get('href', None)
        self_link = coredata.find('dtd:link[@rel="self"]',
                                  ns).get('href', None)
        cite_link = coredata.find('dtd:link[@rel="cited-by"]',
                                  ns)
        if cite_link:
            cite_link = cite_link.get('href', None)
        self.scopus_link = sl
        self.self_link = self_link
        self.cite_link = cite_link

        self._authors = [ScopusAuthor(author) for author in authors]
        self._affiliations = [ScopusAffiliation(aff) for aff
                              in results.findall('dtd:affiliation', ns)]

        if items is not None:<commit_msg>Add property for srctype<commit_after>from . import ns, get_encoded_text, MY_API_KEY
import requests
import xml.etree.ElementTree as ET
import os
import sys

SCOPUS_XML_DIR = os.path.expanduser('~/.scopus/xml')
SCOPUS_ISSN_DIR = os.path.expanduser('~/.scopus/issn')

if not os.path.exists(SCOPUS_XML_DIR):
    os.makedirs(SCOPUS_XML_DIR)

if not os.path.exists(SCOPUS_ISSN_DIR):
    os.makedirs(SCOPUS_ISSN_DIR)


class ScopusAbstract(object):
    """Class to represent the results from a Scopus abstract.

    The results are retrieved by the EID from a query. The results
    are cached in a folder ~/.scopus/xml/{eid}.
    """

    @property
    def url(self):
        """URL to the abstract."""
        return self._url

    @property
    def doi(self):
        """Abstract DOI."""
        return self._doi

    @property
    def title(self):
        """Abstract title."""
        return self._title

    @property
    def aggregationType(self):
        """Type of abstract."""
        return self._aggregationType

    @property
    def publicationName(self):
        """Name of source the abstract is published in."""
        return self._publicationName

    @property
    def srctype(self):
        """Type of source the abstract is published in."""
        return self._srctype

    @property
    def citedby_count(self):
        """Number of times the abstract has been cited."""
        return int(self._citedby_count)

    @property
    def publisher(self):
        """Name of the publisher of the abstract."""
        return self._publisher

    @property
    def source_id(self):
        """Scopus source_id of the abstract."""
        return self._source_id

    @property
    def issn(self):
        """ISSN of the publisher."""
        return self._issn

    @property
    def volume(self):
        """Volume for the abstract."""
        return self._volume

    @property
    def issueIdentifier(self):
        """Issue number for abstract."""
        return self._issueIdentifier

    @property
    def article_number(self):
        """Article number."""
        return self._article_number

    @property
    def startingPage(self):
        """Starting page."""
        return self._startingPage

    @property
    def endingPage(self):
        """Ending page."""
        return self._endingPage

    @property
    def pageRange(self):
        """Page range."""
        return self._pageRange

    @property
    def coverDate(self):
        """The date of the cover the abstract is in."""
        return self._coverDate

    @property
    def authors(self):
        """A list of scopus_api.ScopusAuthor objects."""
        return self._authors

    @property
    def affiliations(self):
        """A list of scopus_api.ScopusAffiliation objects."""
        return self._affiliations

    @property
    def nauthors(self):
        """Return number of authors listed in the abstract."""
        return len(self.authors)

    @property
    def refcount(self):
        """Return number of references of an article, if possible."""
        if self._references is not None:
            return self._references.attrib['refcount']
        else:
            raise TypeError("Could not load article references. "
                            "Did you load with view=FULL?")

    @property
    def references(self):
        """Return EIDs of references of an article."""
        if self._references is not None:
            eids = [r.find("ref-info/refd-itemidlist/itemid", ns).text for r
                    in self._references.findall("reference", ns)]
            return ["2-s2.0-" + eid for eid in eids]
        else:
            raise TypeError("Could not load article references. "
                            "Did you load with view=FULL?")

    def __init__(self, EID, view='META_ABS', refresh=False):
        """Initialize a Scopus abstract

        EID -- The scopus id for an abstract and should be a string.

        view -- (default=META_ABS) The view of the file that should be
        downloaded.  Currently supported values: META, META_ABS, FULL.

        refresh -- (default=False) A boolean that determines if the result
        should be downloaded again.
        """
        allowed_views = ('META', 'META_ABS', 'FULL')
        if view not in allowed_views:
            raise ValueError('view parameter must be one of ' +\
                             ', '.join(allowed_views))

        fEID = os.path.join(SCOPUS_XML_DIR, EID)
        self.file = fEID

        if os.path.exists(fEID) and not refresh:
            with open(fEID) as f:
                text = f.read()
                self.xml = text
                results = ET.fromstring(text)
        else:
            url = "http://api.elsevier.com/content/abstract/eid/{}".format(EID)

            resp = requests.get(url,
                                headers={'Accept': 'application/xml',
                                         'X-ELS-APIKey': MY_API_KEY},
                                params={'view': view})
            self.xml = resp.text
            with open(fEID, 'w') as f:
                if sys.version_info[0] == 3:
                    f.write(self.xml)
                else:
                    f.write(self.xml.encode('utf-8'))

            results = ET.fromstring(resp.text.encode('utf-8'))

        coredata = results.find('dtd:coredata', ns)
        authors = results.find('dtd:authors', ns)
        items = results.find('item', ns)
        self.results = results
        if results.tag == 'service-error':
            raise Exception('\n{0}\n{1}'.format(EID, self.xml))

        self.coredata = coredata
        self.authors_xml = authors
        self._url = get_encoded_text(coredata, 'prism:url')

        self.identifier = get_encoded_text(coredata, 'dc:identifier')
        self.eid = get_encoded_text(coredata, 'dtd:eid')
        self._doi = get_encoded_text(coredata, 'prism:doi')
        self._title = get_encoded_text(coredata, 'dc:title')

        self._aggregationType = get_encoded_text(coredata,
                                                 'prism:aggregationType')
        self._publicationName = get_encoded_text(coredata,
                                                 'prism:publicationName')
        self._srctype = get_encoded_text(coredata, 'dtd:srctype')
        self._citedby_count = get_encoded_text(coredata, 'dtd:citedby-count')
        self._publisher = get_encoded_text(coredata, 'dc:publisher')
        self._source_id = get_encoded_text(coredata, 'dtd:source-id')
        self._issn = get_encoded_text(coredata, 'prism:issn')
        self._volume = get_encoded_text(coredata, 'prism:volume')
        self._issueIdentifier = get_encoded_text(coredata,
                                                 'prism:issueIdentifier')
        self._article_number = get_encoded_text(coredata, 'dtd:article-number')
        self._startingPage = get_encoded_text(coredata, 'prism:startingPage')
        self._endingPage = get_encoded_text(coredata, 'prism:endingPage')
        self._pageRange = get_encoded_text(coredata, 'prism:pageRange')
        self._coverDate = get_encoded_text(coredata, 'prism:coverDate')
        self.creator = get_encoded_text(coredata, 'dc:creator')
        self.description = get_encoded_text(coredata, 'dc:description')

        sl = coredata.find('dtd:link[@rel="scopus"]', ns).get('href', None)
        self_link = coredata.find('dtd:link[@rel="self"]',
                                  ns).get('href', None)
        cite_link = coredata.find('dtd:link[@rel="cited-by"]',
                                  ns)
        if cite_link:
            cite_link = cite_link.get('href', None)
        self.scopus_link = sl
        self.self_link = self_link
        self.cite_link = cite_link

        self._authors = [ScopusAuthor(author) for author in authors]
        self._affiliations = [ScopusAffiliation(aff) for aff
                              in results.findall('dtd:affiliation', ns)]

        if items is not None:
{'fraction_non_alphanumeric': 0.10983316481294236, 'fraction_numerical': 0.0011375126390293225, 'mean_word_length': 2.0980814408770554, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '728121', 'n_tokens_mistral': 4654, 'n_tokens_neox': 4473, 'n_words': 1111}