<commit_before>#!/usr/bin/env python
# -*- coding: utf8 -*-
#
# Copyright (C) 2011 <NAME> <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
:module: mom.net.scheme.dataurl
:synopsis: Makes working with Data-URL schemes easier.
:see: http://en.wikipedia.org/wiki/Data_URI_scheme
:see: https://tools.ietf.org/html/rfc2397

.. autofunction:: dataurl_encode
.. autofunction:: dataurl_decode
"""


from __future__ import absolute_import
from mom.builtins import is_bytes, b

try:
    # Python 3.
    from urllib.parse import quote_from_bytes as quote, \
        unquote_to_bytes as unquote
    from functools import partial
except ImportError:
    # Python 2.5+
    from urllib import quote, unquote

from mom.net.mimeparse import parse_mime_type
from mom.codec import base64_encode, base64_decode


__all__ = [
    "dataurl_encode",
    "dataurl_decode",
]


def dataurl_encode(raw_bytes,
                   mime_type=b('text/plain'),
                   charset=b('US-ASCII'),
                   encoder="base64"):
    """
    Encodes raw bytes into a data URL scheme string.

    :param raw_bytes:
        Raw bytes
    :param mime_type:
        The mime type, e.g. b"text/css" or b"image/png". Default b"text/plain".
    :param charset:
        b"utf-8" if you want the data URL to contain a b"charset=utf-8"
        component. Default b'US-ASCII'. This does not mean however, that your
        raw_bytes will be encoded by this function. You must ensure that
        if you specify, b"utf-8" (or anything else) as the encoding, you
        have encoded your raw data appropriately.
    :param encoder:
        "base64" or None.
    :returns:
        Data URL.
    """
    if not is_bytes(raw_bytes):
        raise TypeError(
            "only raw bytes can be encoded: got %r" % type(raw_bytes).__name__
        )
    if encoder == "base64":
        encode = base64_encode
        codec = b(";base64,")
    else:
        # We want ASCII bytes.
        encode = lambda data: quote(data).encode('ascii')
        codec = b(",")
    mime_type = mime_type or b("")

    charset = b(";charset=") + charset if charset else b("")
    encoded = encode(raw_bytes)
    return b('').join((b("data:"), mime_type, charset, codec, encoded))


def dataurl_decode(data_url):
    """
    Decodes a data URL into raw bytes and metadata.

    :param data_url:
        The data url string.
        If a mime-type definition is missing in the metadata,
        "text/plain;charset=US-ASCII" will be used as default mime-type.
    :returns:
        A 2-tuple::
            (bytes, mime_type)

        See :func:`mom.http.mimeparse.parse_mime_type` for what ``mime_type``
        looks like.
    """
    metadata, encoded = data_url.rsplit(b(","), 1)
    _, metadata = metadata.split(b("data:"), 1)<commit_msg>Encoded data URLs are ASCII-encoded bytes.<commit_after>#!/usr/bin/env python
# -*- coding: utf8 -*-
#
# Copyright (C) 2011 <NAME> <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
:module: mom.net.scheme.dataurl
:synopsis: Makes working with Data-URL schemes easier.
:see: http://en.wikipedia.org/wiki/Data_URI_scheme
:see: https://tools.ietf.org/html/rfc2397

.. autofunction:: dataurl_encode
.. autofunction:: dataurl_decode
"""


from __future__ import absolute_import
from mom.builtins import is_bytes, b

try:
    # Python 3.
    from urllib.parse import quote_from_bytes as quote, \
        unquote_to_bytes as unquote
    from functools import partial
except ImportError:
    # Python 2.5+
    from urllib import quote, unquote

from mom.net.mimeparse import parse_mime_type
from mom.codec import base64_encode, base64_decode


__all__ = [
    "dataurl_encode",
    "dataurl_decode",
]


def dataurl_encode(raw_bytes,
                   mime_type=b('text/plain'),
                   charset=b('US-ASCII'),
                   encoder="base64"):
    """
    Encodes raw bytes into a data URL scheme string.

    :param raw_bytes:
        Raw bytes
    :param mime_type:
        The mime type, e.g. b"text/css" or b"image/png". Default b"text/plain".
    :param charset:
        b"utf-8" if you want the data URL to contain a b"charset=utf-8"
        component. Default b'US-ASCII'. This does not mean however, that your
        raw_bytes will be encoded by this function. You must ensure that
        if you specify, b"utf-8" (or anything else) as the encoding, you
        have encoded your raw data appropriately.
    :param encoder:
        "base64" or None.
    :returns:
        Data URL.
    """
    if not is_bytes(raw_bytes):
        raise TypeError(
            "only raw bytes can be encoded: got %r" % type(raw_bytes).__name__
        )
    if encoder == "base64":
        encode = base64_encode
        codec = b(";base64,")
    else:
        # We want ASCII bytes.
        encode = lambda data: quote(data).encode('ascii')
        codec = b(",")
    mime_type = mime_type or b("")

    charset = b(";charset=") + charset if charset else b("")
    encoded = encode(raw_bytes)
    return b('').join((b("data:"), mime_type, charset, codec, encoded))


def dataurl_decode(data_url):
    """
    Decodes a data URL into raw bytes and metadata.

    :param data_url:
        The data url string.
        If a mime-type definition is missing in the metadata,
        "text/plain;charset=US-ASCII" will be used as default mime-type.
    :returns:
        A 2-tuple::
            (bytes, mime_type)

        See :func:`mom.http.mimeparse.parse_mime_type` for what ``mime_type``
        looks like.
    """
    if not is_bytes(data_url):
        raise TypeError(
            "data URLs must be ASCII-encoded bytes: got %r" %
            type(raw_bytes).__name__
        )
    metadata, encoded = data_url.rsplit(b(","), 1)
    _, metadata = metadata.split(b("data:"), 1)
{'fraction_non_alphanumeric': 0.1066902446212791, 'fraction_numerical': 0.010610079575596816, 'mean_word_length': 2.7373348017621146, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '325746', 'n_tokens_mistral': 2162, 'n_tokens_neox': 2044, 'n_words': 700}