<commit_before>from collections import OrderedDict

import requests

from civis import APIClient
from civis.base import EmptyResultError
from civis.utils._deprecation import deprecate_param
try:
    from requests_toolbelt.multipart.encoder import MultipartEncoder
    HAS_TOOLBELT = True
except ImportError:
    HAS_TOOLBELT = False


@deprecate_param('v2.0.0', 'api_key')
def file_to_civis(buf, name, api_key=None, client=None, **kwargs):
    """Upload a file to Civis.

    Parameters
    ----------
    buf : file-like object
        The file or other buffer that you wish to upload.
    name : str
        The name you wish to give the file.
    api_key : DEPRECATED str, optional
        Your Civis API key. If not given, the :envvar:`CIVIS_API_KEY`
        environment variable will be used.
    client : :class:`civis.APIClient`, optional
        If not provided, an :class:`civis.APIClient` object will be
        created from the :envvar:`CIVIS_API_KEY`.
    **kwargs : kwargs
        Extra keyword arguments will be passed to the file creation
        endpoint. See :func:`~civis.resources._resources.Files.post`.

    Returns
    -------
    file_id : int
        The new Civis file ID.

    Examples
    --------
    >>> # Upload file which expires in 30 days
    >>> with open("my_data.csv", "r") as f:
    ...     file_id = file_to_civis(f, 'my_data')
    >>> # Upload file which never expires
    >>> with open("my_data.csv", "r") as f:
    ...     file_id = file_to_civis(f, 'my_data', expires_at=None)

    Notes
    -----
    If you are opening a binary file (e.g., a compressed archive) to
    pass to this function, do so using the ``'rb'`` (read binary)
    mode (e.g., ``open('myfile.zip', 'rb')``).

    If you have the `requests-toolbelt` package installed
    (`pip install requests-toolbelt`), then this function will stream
    from the open file pointer into Platform. If `requests-toolbelt`
    is not installed, then it will need to read the entire buffer
    into memory before writing.
    """
    if client is None:
        client = APIClient(api_key=api_key)
    file_response = client.files.post(name, **kwargs)

    # Platform has given us a URL to which we can upload a file.
    # The file must be uploaded with a POST formatted as per
    # http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-post-example.html
    # Note that the payload must have "key" first and "file" last.
    form = file_response.upload_fields
    form_key = OrderedDict(key=form.pop('key'))
    form_key.update(form)
    form_key['file'] = buf

    url = file_response.upload_url
    if HAS_TOOLBELT:
        # This streams from the open file buffer without holding the
        # contents in memory.
        en = MultipartEncoder(fields=form_key)
        if en.len / 2 ** 20 < 100:
            # Semi-arbitrary cutoff for "small" files.
            # Send these with requests directly because that uses less CPU
            response = requests.post(url, files=form_key)
        else:
            response = requests.post(url, data=en,
                                     headers={'Content-Type': en.content_type})
    else:
        response = requests.post(url, files=form_key)
    response.raise_for_status()

    return file_response.id
<commit_msg>ENH: Error on file too large.<commit_after>from collections import OrderedDict

import requests

from civis import APIClient
from civis.base import EmptyResultError
from civis.utils._deprecation import deprecate_param
try:
    from requests_toolbelt.multipart.encoder import MultipartEncoder
    HAS_TOOLBELT = True
except ImportError:
    HAS_TOOLBELT = False


@deprecate_param('v2.0.0', 'api_key')
def file_to_civis(buf, name, api_key=None, client=None, **kwargs):
    """Upload a file to Civis.

    Parameters
    ----------
    buf : file-like object
        The file or other buffer that you wish to upload.
    name : str
        The name you wish to give the file.
    api_key : DEPRECATED str, optional
        Your Civis API key. If not given, the :envvar:`CIVIS_API_KEY`
        environment variable will be used.
    client : :class:`civis.APIClient`, optional
        If not provided, an :class:`civis.APIClient` object will be
        created from the :envvar:`CIVIS_API_KEY`.
    **kwargs : kwargs
        Extra keyword arguments will be passed to the file creation
        endpoint. See :func:`~civis.resources._resources.Files.post`.

    Returns
    -------
    file_id : int
        The new Civis file ID.

    Examples
    --------
    >>> # Upload file which expires in 30 days
    >>> with open("my_data.csv", "r") as f:
    ...     file_id = file_to_civis(f, 'my_data')
    >>> # Upload file which never expires
    >>> with open("my_data.csv", "r") as f:
    ...     file_id = file_to_civis(f, 'my_data', expires_at=None)

    Notes
    -----
    If you are opening a binary file (e.g., a compressed archive) to
    pass to this function, do so using the ``'rb'`` (read binary)
    mode (e.g., ``open('myfile.zip', 'rb')``).

    If you have the `requests-toolbelt` package installed
    (`pip install requests-toolbelt`), then this function will stream
    from the open file pointer into Platform. If `requests-toolbelt`
    is not installed, then it will need to read the entire buffer
    into memory before writing.
    """
    if client is None:
        client = APIClient(api_key=api_key)
    file_response = client.files.post(name, **kwargs)

    # Platform has given us a URL to which we can upload a file.
    # The file must be uploaded with a POST formatted as per
    # http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-post-example.html
    # Note that the payload must have "key" first and "file" last.
    form = file_response.upload_fields
    form_key = OrderedDict(key=form.pop('key'))
    form_key.update(form)
    form_key['file'] = buf

    url = file_response.upload_url
    if HAS_TOOLBELT:
        # This streams from the open file buffer without holding the
        # contents in memory.
        en = MultipartEncoder(fields=form_key)
        # The refusal error from AWS states 5368730624 is the max size allowed
        if en.len >= 5 * 2 ** 30:  # 5 GB
            msg = "Cannot upload files greater than 5GB. Got {:d}."
            raise ValueError(msg.format(en.len))
        elif en.len <= 100 * 2 ** 20:  # 100 MB
            # Semi-arbitrary cutoff for "small" files.
            # Send these with requests directly because that uses less CPU
            response = requests.post(url, files=form_key)
        else:
            response = requests.post(url, data=en,
                                     headers={'Content-Type': en.content_type})
    else:
        response = requests.post(url, files=form_key)
    response.raise_for_status()

    return file_response.id

{'fraction_non_alphanumeric': 0.10016178849830858, 'fraction_numerical': 0.006618620385350787, 'mean_word_length': 2.5940803382663846, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '187031', 'n_tokens_mistral': 2104, 'n_tokens_neox': 2018, 'n_words': 692}