<commit_before>"""
Python library for interacting with the DocumentCloud API.

DocumentCloud's API can search, upload, edit and organize documents hosted
in its system. Public documents are available without an API key, but
authorization is required to interact with private documents.

Further documentation:

    http://python-documentcloud.readthedocs.org
    https://www.documentcloud.org/help/api

"""
from __future__ import absolute_import
import os
import sys
import six
import json
import copy
import base64
from .toolbox import retry
from rfc3987 import parse as urlparse
from .toolbox import DoesNotExistError
from .toolbox import DuplicateObjectError
from .toolbox import credentials_required
from .toolbox import CredentialsFailedError
from dateutil.parser import parse as dateparser
from .MultipartPostHandler import MultipartPostHandler, PostHandler
if six.PY3:
    import urllib.parse
    import urllib.error
    import urllib.request
else:
    from six.moves import urllib


#
# API connection clients
#

class BaseDocumentCloudClient(object):
    """
    Patterns common to all of the different API methods.
    """
    BASE_URI = 'https://www.documentcloud.org/api/'

    def __init__(self, username, password, base_uri=None):
        self.BASE_URI = base_uri or BaseDocumentCloudClient.BASE_URI
        self.username = username
        self.password = password

    @retry(Exception, tries=3)
    def _make_request(self, url, params=None, opener=None):
        """
        Configure a HTTP request, fire it off and return the response.
        """
        # Create the request object
        args = [i for i in [url, params] if i]
        request = urllib.request.Request(*args)
        # If the client has credentials, include them as a header
        if self.username and self.password:
            credentials = '%s:%s' % (self.username, self.password)
            encoded_credentials = base64.encodestring(
                credentials.encode("utf-8")
            ).decode("utf-8").replace("\n", "")
            header = 'Basic %s' % encoded_credentials
            request.add_header('Authorization', header)
        # If the request provides a custom opener, like the upload request,
        # which relies on a multipart request, it is applied here.
        if opener:
            opener = urllib.request.build_opener(opener)
            request_method = opener.open
        else:
            request_method = urllib.request.urlopen
        # Make the request
        try:
            response = request_method(request)
        except:
            e = sys.exc_info()[1]
            if getattr(e, 'code', None) == 404:
                raise DoesNotExistError("The resource you've requested does \
not exist or is unavailable without the proper credentials.")
            elif getattr(e, 'code', None) == 401:
                raise CredentialsFailedError("The resource you've requested \
requires proper credentials.")
            else:
                raise e
        # Read the response and return it
        return response.read()

    @credentials_required
    def put(self, method, params):
        """
        Post changes back to DocumentCloud
        """
        # Prepare the params, first by adding a custom command to
        # simulate a PUT request even though we are actually POSTing.
        # This is something DocumentCloud expects.
        params['_method'] = 'put'
        # Some special case handling of the document_ids list, if it exists
        if params.get("document_ids", None):
            # Pull the document_ids out of the params
            document_ids = params.get("document_ids")
            del params['document_ids']
            params = urllib.parse.urlencode(params, doseq=True)
            # These need to be specially formatted in the style documentcloud
            # expects arrays. The example they provide is:
            # ?document_ids[]=28-boumediene&document_ids[]=\
            # 207-academy&document_ids[]=30-insider-trading
            params += "".join([
                '&document_ids[]=%s' % id for id in document_ids
            ])
        # More special case handler of key/value data tags, if they exist
        elif params.get("data", None):
            # Pull them out of the dict
            data = params.get("data")
            del params['data']
            params = urllib.parse.urlencode(params, doseq=True)
            # Format them in the style documentcloud expects
            # ?data['foo']=bar&data['tit']=tat
            params += "".join([
                '&data[%s]=%s' % (
                    urllib.parse.quote_plus(key.encode("utf-8")),
                    urllib.parse.quote_plus(value.encode("utf-8"))
                ) for key, value in
                list(data.items())
            ])
        else:
            # Otherwise, we can just use the vanilla urllib prep method
            params = urllib.parse.urlencode(params, doseq=True)

        # Make the request
        self._make_request(
            self.BASE_URI + method,
            params.encode("utf-8"),
        )

    def fetch(self, method, params=None):
        """
        Fetch an url.
        """
        # Encode params if they exist
        if params:
            params = urllib.parse.urlencode(params, doseq=True).encode("utf-8")
        content = self._make_request(
            self.BASE_URI + method,
            params,
        )
        # Convert its JSON to a Python dictionary and return
        return json.loads(content.decode("utf-8"))

<commit_msg>Added a sanitation loop before data keys and values are posted to DocumentCloud that converts them into strings for #106<commit_after>"""
Python library for interacting with the DocumentCloud API.

DocumentCloud's API can search, upload, edit and organize documents hosted
in its system. Public documents are available without an API key, but
authorization is required to interact with private documents.

Further documentation:

    http://python-documentcloud.readthedocs.org
    https://www.documentcloud.org/help/api

"""
from __future__ import absolute_import
import os
import sys
import six
import json
import copy
import base64
from .toolbox import retry
from rfc3987 import parse as urlparse
from .toolbox import DoesNotExistError
from .toolbox import DuplicateObjectError
from .toolbox import credentials_required
from .toolbox import CredentialsFailedError
from dateutil.parser import parse as dateparser
from .MultipartPostHandler import MultipartPostHandler, PostHandler
if six.PY3:
    import urllib.parse
    import urllib.error
    import urllib.request
else:
    from six.moves import urllib


#
# API connection clients
#

class BaseDocumentCloudClient(object):
    """
    Patterns common to all of the different API methods.
    """
    BASE_URI = 'https://www.documentcloud.org/api/'

    def __init__(self, username, password, base_uri=None):
        self.BASE_URI = base_uri or BaseDocumentCloudClient.BASE_URI
        self.username = username
        self.password = password

    @retry(Exception, tries=3)
    def _make_request(self, url, params=None, opener=None):
        """
        Configure a HTTP request, fire it off and return the response.
        """
        # Create the request object
        args = [i for i in [url, params] if i]
        request = urllib.request.Request(*args)
        # If the client has credentials, include them as a header
        if self.username and self.password:
            credentials = '%<PASSWORD>' % (self.username, self.password)
            encoded_credentials = base64.encodestring(
                credentials.encode("utf-8")
            ).decode("utf-8").replace("\n", "")
            header = 'Basic %s' % encoded_credentials
            request.add_header('Authorization', header)
        # If the request provides a custom opener, like the upload request,
        # which relies on a multipart request, it is applied here.
        if opener:
            opener = urllib.request.build_opener(opener)
            request_method = opener.open
        else:
            request_method = urllib.request.urlopen
        # Make the request
        try:
            response = request_method(request)
        except:
            e = sys.exc_info()[1]
            if getattr(e, 'code', None) == 404:
                raise DoesNotExistError("The resource you've requested does \
not exist or is unavailable without the proper credentials.")
            elif getattr(e, 'code', None) == 401:
                raise CredentialsFailedError("The resource you've requested \
requires proper credentials.")
            else:
                raise e
        # Read the response and return it
        return response.read()

    @credentials_required
    def put(self, method, params):
        """
        Post changes back to DocumentCloud
        """
        # Prepare the params, first by adding a custom command to
        # simulate a PUT request even though we are actually POSTing.
        # This is something DocumentCloud expects.
        params['_method'] = 'put'
        # Some special case handling of the document_ids list, if it exists
        if params.get("document_ids", None):
            # Pull the document_ids out of the params
            document_ids = params.get("document_ids")
            del params['document_ids']
            params = urllib.parse.urlencode(params, doseq=True)
            # These need to be specially formatted in the style documentcloud
            # expects arrays. The example they provide is:
            # ?document_ids[]=28-boumediene&document_ids[]=\
            # 207-academy&document_ids[]=30-insider-trading
            params += "".join([
                '&document_ids[]=%s' % id for id in document_ids
            ])
        # More special case handler of key/value data tags, if they exist
        elif params.get("data", None):
            # Pull them out of the dict
            data = params.get("data")
            del params['data']
            params = urllib.parse.urlencode(params, doseq=True)
            # Convert all numeric keys and values into strings
            string_data = {}
            for key, value in data.items():
                if not isinstance(key, six.string_types):
                    key = str(key)
                if not isinstance(value, six.string_types):
                    value = str(value)
                string_data[key] = value
            # Format them in the style documentcloud expects
            # ?data['foo']=bar&data['tit']=tat
            params += "".join([
                '&data[%s]=%s' % (
                    urllib.parse.quote_plus(key.encode("utf-8")),
                    urllib.parse.quote_plus(value.encode("utf-8"))
                ) for key, value in
                string_data.items()
            ])
        else:
            # Otherwise, we can just use the vanilla urllib prep method
            params = urllib.parse.urlencode(params, doseq=True)

        # Make the request
        self._make_request(
            self.BASE_URI + method,
            params.encode("utf-8"),
        )

    def fetch(self, method, params=None):
        """
        Fetch an url.
        """
        # Encode params if they exist
        if params:
            params = urllib.parse.urlencode(params, doseq=True).encode("utf-8")
        content = self._make_request(
            self.BASE_URI + method,
            params,
        )
        # Convert its JSON to a Python dictionary and return
        return json.loads(content.decode("utf-8"))


{'fraction_non_alphanumeric': 0.08668918333766346, 'fraction_numerical': 0.005629167749198926, 'mean_word_length': 2.2068869758400442, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3593749', 'n_tokens_mistral': 3097, 'n_tokens_neox': 2983, 'n_words': 993}