<commit_before># -*- coding: utf-8 -*-
"""
Base class for EDGAR filing.

:copyright: © 2019 by <NAME>
:license: MIT, see LICENSE for more details.
"""

# import os
import re
import logging

from pyedgar import config
from pyedgar.utilities import edgarweb
from pyedgar.utilities import forms
from pyedgar.utilities.forms import FORMS
from pyedgar.utilities import localstore

class Filing(object):
    """
    Base class for EDGAR filing, with format aware parsing
    to provide access to filing documents as objects.
    """
    #: Firm CIK
    _cik = None
    #: Filing Accession
    _accession = None
    #: Whether filings are cached locally, or to get them from EDGAR
    _local_cache = False
    #: The 'loose' form type, with some family hierarchy: 10s, Def 14s, etc.
    _type = None
    #: The exact form type, extracted from the main <TYPE> tag.
    _type_exact = None
    #: Paths to local filing. Set lazily.
    _filing_local_path = None
    #: Paths to remote filing. Set lazily.
    _filing_url = None
    #: Form Data (main headers)
    _headers = {}
    #: Full text of the document, loaded lazily.
    _full_text = None
    #: Array of filed exhibits, parsed lazily.
    _documents = None
    #: Object logger for logging things and stuff.
    __log = None
    #: Read-filing arguments
    read_args = None

    def __init__(self, cik=None, accession=None, **kwargs):
        """
        Initialization sets CIK, Accession, and optionally
        loads filing from disk.

        Args:
            cik (int,str): Numeric CIK for firm.
            accession (str): Accession for filing, filed under CIK.
                Expected format: 0123456789-01-012345, or 012345678901012345.

        Returns:
            Filing object.

        Raises:
            None. Loading done lazily.
        """
        self.__log = logging.getLogger('pyedgar.filing.Filing')

        self._set_cik(cik)
        self._set_accession(accession)

        self._local_cache = config.CACHE_FEED

        self.read_args = kwargs

    def __repr__(self):
        return ("<EDGAR filing ({}/{}) Headers:{}, Text:{}, Documents:{}>"
                .format(self.cik, self.accession,
                        bool(self._headers),
                        bool(self._full_text),
                        bool(self._documents),))

    def __str__(self):
        return self.__repr__()

    def _set_cik(self, cik=None):
        """
        Set cik on object, verifying format is CIK-like.

        Args:
            cik (int,str): Numeric CIK for firm, either as int or string.

        Returns:
            int: CIK of the filing.

        Raises:
            ValueError: if cik is not numerical (or castable to int).
        """
        try:
            if cik is not None:
                self._cik = int(cik)
        except ValueError:
            # They didn't pass in a CIK that looked like a number
            raise ValueError("CIKs must be numeric variables,"
                             " you passed in {}".format(cik))

        return self._cik

    def _set_accession(self, accession=None):
        """
        Set accession on object, verifying format is accession-like.
        Expected format: 0123456789-01-012345, or 012345678901012345.
        Function converts the latter to the former.

        Args:
            accession (str): Accession number as a string, with format
                0123456789-01-012345, or 012345678901012345.

        Returns:
            The formatted Accession as a string of 20 characters.

        Raises:
            ValueError: if accession is not expected format.
        """
        try:
            if accession and localstore.ACCESSION_RE.search(accession):
                if len(accession) == 18:
                    self._accession = '{}-{}-{}'.format(accession[:10],
                                                        accession[10:12],
                                                        accession[12:])
                else:
                    self._accession = accession
        except TypeError:
            # They didn't pass in an accession that was a string.
            raise ValueError("Accessions must be 18/20 character strings of format"
                             " ##########-##-######, you passed in {}"
                             .format(accession))

        return self.accession

    def _set_full_text(self):
        """
        Full text of the filing at cik/accession.
        Lazily load the full text of the filing into memory.

        TODO: Handle non-SGML headers (e.g. from S3)

        Returns:
            String representing the full text of the EDGAR filing.

        Raises:
            FileNotFoundError: The file wasn't found in the local cache.
        """
        if not self._full_text:
            self.__log.debug("Local cache: %r", self._local_cache)
            if self._local_cache:
                self.__log.debug("Local cache if: %r", self._local_cache)
                try:
                    self._full_text = forms.get_full_filing(
                        self.path, **self.read_args)
                except FileNotFoundError:
                    msg = ("Filing not found for CIK:{} / Accession:{}"
                        .format(self.cik, self.accession))
                    self.__log.debug(msg)
                    raise FileNotFoundError(msg)
            else:
                self.__log.debug("Local cache else: %r", self._local_cache)
                self._full_text = edgarweb.download_form_from_web(
                    self.cik, self.accession)

        return self._full_text

    def _set_headers(self, set_flat=True, set_hierarchical=True):
        """
        Load the full set of headers of the filing at cik/accession into memory.<commit_msg>Add web DL fallback to filing<commit_after># -*- coding: utf-8 -*-
"""
Base class for EDGAR filing.

:copyright: © 2020 by <NAME>
:license: MIT, see LICENSE for more details.
"""

# import os
import re
import logging

from pyedgar import config
from pyedgar.utilities import edgarweb
from pyedgar.utilities import forms
from pyedgar.utilities.forms import FORMS
from pyedgar.utilities import localstore

class Filing(object):
    """
    Base class for EDGAR filing, with format aware parsing
    to provide access to filing documents as objects.
    """
    #: Firm CIK
    _cik = None
    #: Filing Accession
    _accession = None
    #: Whether filings are cached locally, or to get them from EDGAR
    _local_cache = False
    #: If filing is not found, fallback to downloading from the web.
    _web_fallback = True
    #: The 'loose' form type, with some family hierarchy: 10s, Def 14s, etc.
    _type = None
    #: The exact form type, extracted from the main <TYPE> tag.
    _type_exact = None
    #: Paths to local filing. Set lazily.
    _filing_local_path = None
    #: Paths to remote filing. Set lazily.
    _filing_url = None
    #: Form Data (main headers)
    _headers = {}
    #: Full text of the document, loaded lazily.
    _full_text = None
    #: Array of filed exhibits, parsed lazily.
    _documents = None
    #: Object logger for logging things and stuff.
    __log = None
    #: Read-filing arguments
    read_args = None

    def __init__(self, cik=None, accession=None,
                 use_cache=None, web_fallback=True, **kwargs):
        """
        Initialization sets CIK, Accession, and optionally
        loads filing from disk.

        Args:
            cik (int,str): Numeric CIK for firm.
            accession (str): Accession for filing, filed under CIK.
                Expected format: 0123456789-01-012345, or 012345678901012345.
            use_cache (bool): Use the local cache at `config.FEED_CACHE_ROOT`,
                default to `config.CACHE_FEED`.
            web_fallback (bool): When FileNotFoundError is raised, try
                `edgarweb.download_form_from_web` instead.

        Returns:
            Filing object.

        Raises:
            None. Loading done lazily.
        """
        self.__log = logging.getLogger('pyedgar.filing.Filing')

        self._set_cik(cik)
        self._set_accession(accession)

        self._local_cache = use_cache if use_cache is not None else config.CACHE_FEED
        self._web_fallback = web_fallback

        self.read_args = kwargs

    def __repr__(self):
        return ("<EDGAR filing ({}/{}) Headers:{}, Text:{}, Documents:{}>"
                .format(self.cik, self.accession,
                        bool(self._headers),
                        bool(self._full_text),
                        bool(self._documents),))

    def __str__(self):
        return self.__repr__()

    def _set_cik(self, cik=None):
        """
        Set cik on object, verifying format is CIK-like.

        Args:
            cik (int,str): Numeric CIK for firm, either as int or string.

        Returns:
            int: CIK of the filing.

        Raises:
            ValueError: if cik is not numerical (or castable to int).
        """
        try:
            if cik is not None:
                self._cik = int(cik)
        except ValueError:
            # They didn't pass in a CIK that looked like a number
            raise ValueError("CIKs must be numeric variables,"
                             " you passed in {}".format(cik))

        return self._cik

    def _set_accession(self, accession=None):
        """
        Set accession on object, verifying format is accession-like.
        Expected format: 0123456789-01-012345, or 012345678901012345.
        Function converts the latter to the former.

        Args:
            accession (str): Accession number as a string, with format
                0123456789-01-012345, or 012345678901012345.

        Returns:
            The formatted Accession as a string of 20 characters.

        Raises:
            ValueError: if accession is not expected format.
        """
        try:
            if accession and localstore.ACCESSION_RE.search(accession):
                if len(accession) == 18:
                    self._accession = '{}-{}-{}'.format(accession[:10],
                                                        accession[10:12],
                                                        accession[12:])
                else:
                    self._accession = accession
        except TypeError:
            # They didn't pass in an accession that was a string.
            raise ValueError("Accessions must be 18/20 character strings of format"
                             " ##########-##-######, you passed in {}"
                             .format(accession))

        return self.accession

    def _set_full_text(self):
        """
        Full text of the filing at cik/accession.
        Lazily load the full text of the filing into memory.
        Use cache if `self._local_cache`, and fall back to EDGAR website
        if `self._web_fallback`.

        Returns:
            String representing the full text of the EDGAR filing.

        Raises:
            FileNotFoundError: The file wasn't found in the local cache if
                `self._local_cache`.
        """
        if not self._full_text:
            self.__log.debug("Local cache: %r", self._local_cache)
            if self._local_cache:
                self.__log.debug("Local cache if: %r", self._local_cache)
                try:
                    self._full_text = forms.get_full_filing(
                        self.path, **self.read_args)
                    return self._full_text
                except FileNotFoundError:
                    msg = ("Filing not found for CIK:{} / Accession:{}"
                        .format(self.cik, self.accession))
                    self.__log.debug(msg)
                    if not self._web_fallback:
                        raise FileNotFoundError(msg)

            self.__log.debug("Downloading from EDGAR web: %d/%s",
                             self.cik, self.accession)
            self._full_text = edgarweb.download_form_from_web(self.cik,
                                                              self.accession)

        return self._full_text

    def _set_headers(self, set_flat=True, set_hierarchical=True):
        """
        Load the full set of headers of the filing at cik/accession into memory.
{'fraction_non_alphanumeric': 0.08443937790082241, 'fraction_numerical': 0.021740900578128816, 'mean_word_length': 1.7396832478251172, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3901759', 'n_tokens_mistral': 3644, 'n_tokens_neox': 3258, 'n_words': 1076}