<commit_before># emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the duecredit package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##

# Just for testing of robust operation
import os
if 'DUECREDIT_TEST_EARLY_IMPORT_ERROR' in os.environ.keys():
    raise ImportError("DUECREDIT_TEST_EARLY_IMPORT_ERROR")

import locale
import time
from collections import defaultdict, Iterator
import copy
from os.path import dirname, exists
import pickle
import requests
import tempfile
from six import PY2, itervalues, iteritems
import warnings

from .config import CACHE_DIR, DUECREDIT_FILE
from .entries import BibTeX, Doi
from .log import lgr

_PREFERRED_ENCODING = locale.getpreferredencoding()

def get_doi_cache_file(doi):
    return os.path.join(CACHE_DIR, doi)


def import_doi(doi, sleep=0.5, retries=10):
    cached = get_doi_cache_file(doi)

    if exists(cached):
        with open(cached) as f:
            doi = f.read()
            if PY2:
                return doi.decode('utf-8')
            return doi

    # else -- fetch it
    #headers = {'Accept': 'text/bibliography; style=bibtex'}
    headers = {'Accept': 'application/x-bibtex; charset=utf-8'}
    url = 'http://dx.doi.org/' + doi
    while retries > 0:
        r = requests.get(url, headers=headers)
        r.encoding = 'UTF-8'
        bibtex = r.text.strip()
        if bibtex.startswith('@'):
            # no more retries necessary
            break
        lgr.warning("Failed to obtain bibtex from doi.org, retrying...")
        time.sleep(sleep)  # give some time to the server
        retries -= 1
    status_code = r.status_code
    if not bibtex.startswith('@'):
        raise ValueError('Query %(url)s for BibTeX for a DOI %(doi)s (wrong doi?) has failed. '
                         'Response code %(status_code)d. '
                         #'BibTeX response was: %(bibtex)s'
                         % locals())
    if not exists(cached):
        cache_dir = dirname(cached)
        if not exists(cache_dir):
            os.makedirs(cache_dir)
        with open(cached, 'w') as f:
            if PY2:
                f.write(bibtex.encode('utf-8'))
            else:
                f.write(bibtex)
    return bibtex


class EnumeratedEntries(Iterator):
    """A container of entries enumerated referenced by their entry_key"""
    def __init__(self):
        self._keys2refnr = {}
        self._refnr2keys = {}
        self._refnr = 1

    def add(self, entry_key):
        """Add entry_key and update refnr"""
        if entry_key not in self._keys2refnr:
            self._keys2refnr[entry_key] = self._refnr
            self._refnr2keys[self._refnr] = entry_key
            self._refnr += 1

    def __getitem__(self, item):
        if item not in self._keys2refnr:
            raise KeyError('{0} not present'.format(item))
        return self._keys2refnr[item]

    def fromrefnr(self, refnr):
        if refnr not in self._refnr2keys:
            raise KeyError('{0} not present'.format(refnr))
        return self._refnr2keys[refnr]

    def __iter__(self):
        return iteritems(self._keys2refnr)

    # Python 3
    def __next__(self):
        return self.next()

    def next(self):
        yield next(self.__iter__())

    def __len__(self):
        return len(self._keys2refnr)


class TextOutput(object):  # TODO some parent class to do what...?
    def __init__(self, fd, collector, style=None):
        self.fd = fd
        self.collector = collector
        # TODO: check that CLS style actually exists
        self.style = style
        if 'DUECREDIT_STYLE' in os.environ.keys():
            self.style = os.environ['DUECREDIT_STYLE']
        else:
            self.style = 'harvard1'
<commit_msg>Start refactoring i/o into generic Output class<commit_after># emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the duecredit package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##

# Just for testing of robust operation
import os
if 'DUECREDIT_TEST_EARLY_IMPORT_ERROR' in os.environ.keys():
    raise ImportError("DUECREDIT_TEST_EARLY_IMPORT_ERROR")

import locale
import time
from collections import defaultdict, Iterator
import copy
from os.path import dirname, exists
import pickle
import requests
import tempfile
from six import PY2, itervalues, iteritems
import warnings

from .config import CACHE_DIR, DUECREDIT_FILE
from .entries import BibTeX, Doi
from .log import lgr

_PREFERRED_ENCODING = locale.getpreferredencoding()

def get_doi_cache_file(doi):
    return os.path.join(CACHE_DIR, doi)


def import_doi(doi, sleep=0.5, retries=10):
    cached = get_doi_cache_file(doi)

    if exists(cached):
        with open(cached) as f:
            doi = f.read()
            if PY2:
                return doi.decode('utf-8')
            return doi

    # else -- fetch it
    #headers = {'Accept': 'text/bibliography; style=bibtex'}
    headers = {'Accept': 'application/x-bibtex; charset=utf-8'}
    url = 'http://dx.doi.org/' + doi
    while retries > 0:
        r = requests.get(url, headers=headers)
        r.encoding = 'UTF-8'
        bibtex = r.text.strip()
        if bibtex.startswith('@'):
            # no more retries necessary
            break
        lgr.warning("Failed to obtain bibtex from doi.org, retrying...")
        time.sleep(sleep)  # give some time to the server
        retries -= 1
    status_code = r.status_code
    if not bibtex.startswith('@'):
        raise ValueError('Query %(url)s for BibTeX for a DOI %(doi)s (wrong doi?) has failed. '
                         'Response code %(status_code)d. '
                         #'BibTeX response was: %(bibtex)s'
                         % locals())
    if not exists(cached):
        cache_dir = dirname(cached)
        if not exists(cache_dir):
            os.makedirs(cache_dir)
        with open(cached, 'w') as f:
            if PY2:
                f.write(bibtex.encode('utf-8'))
            else:
                f.write(bibtex)
    return bibtex


class EnumeratedEntries(Iterator):
    """A container of entries enumerated referenced by their entry_key"""
    def __init__(self):
        self._keys2refnr = {}
        self._refnr2keys = {}
        self._refnr = 1

    def add(self, entry_key):
        """Add entry_key and update refnr"""
        if entry_key not in self._keys2refnr:
            self._keys2refnr[entry_key] = self._refnr
            self._refnr2keys[self._refnr] = entry_key
            self._refnr += 1

    def __getitem__(self, item):
        if item not in self._keys2refnr:
            raise KeyError('{0} not present'.format(item))
        return self._keys2refnr[item]

    def fromrefnr(self, refnr):
        if refnr not in self._refnr2keys:
            raise KeyError('{0} not present'.format(refnr))
        return self._refnr2keys[refnr]

    def __iter__(self):
        return iteritems(self._keys2refnr)

    # Python 3
    def __next__(self):
        return self.next()

    def next(self):
        yield next(self.__iter__())

    def __len__(self):
        return len(self._keys2refnr)


def _is_contained(toppath, subpath):
    if ':' not in toppath:
        return ((toppath == subpath) or
                (subpath.startswith(toppath + '.')) or
                (subpath.startswith(toppath + ':')))
    else:
        return subpath.startswith(toppath + '.')


class Output(object):
    """A generic class for setting up citations that then will be outputted
    differently (e.g., Bibtex, Text, etc.)"""
    def __init__(self, fd, collector, **kwargs):
        self.fd = fd
        self.collector = collector

    def _filter_citations(self, tags=None):
        """Given all the citations, filter only those that the user wants and
        those that were actually used"""
        if not tags:
            tags = os.environ.get('DUECREDIT_REPORT_TAGS', 'reference-implementation,implementation').split(',')
        tags = set(tags)

        citations = self.collector.citations
        if tags != {'*'}:
            # Filter out citations based on tags
            citations = dict((k, c)
                             for k, c in iteritems(citations)
                             if tags.intersection(c.tags))

        packages = defaultdict(list)
        modules = defaultdict(list)
        objects = defaultdict(list)

        # store the citations according to their path and divide them into
        # the right level
        for (path, entry_key), citation in iteritems(citations):
            if ':' in path:
                objects[path].append(citation)
            elif '.' in path:
                modules[path].append(citation)
            else:
                packages[path].append(citation)

        # now we need to filter out the packages that don't have modules
        # or objects cited, or are specifically requested
        cited_packages = list(packages)
        cited_modobj = list(modules) + list(objects)
        for package in cited_packages:
            package_citations = packages[package]
            if filter(lambda x: x.cite_module, package_citations) or \
                    filter(lambda x: _is_contained(package, x), cited_modobj):
                continue
            else:
                # we don't need it
                del packages[package]

        return packages, modules, objects


class TextOutput(object):  # TODO some parent class to do what...?
    def __init__(self, fd, collector, style=None):
        self.fd = fd
        self.collector = collector
        # TODO: check that CLS style actually exists
        self.style = style
        if 'DUECREDIT_STYLE' in os.environ.keys():
            self.style = os.environ['DUECREDIT_STYLE']
        else:
            self.style = 'harvard1'

{'fraction_non_alphanumeric': 0.12177917115704909, 'fraction_numerical': 0.006858038600960126, 'mean_word_length': 2.2738935214881333, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2433030', 'n_tokens_mistral': 3185, 'n_tokens_neox': 3087, 'n_words': 844}