<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
import os
from os.path import splitext, basename
import warnings  # NOQA
import vtool.exif as vtexif
import utool as ut
#import numpy as np  # NOQA
#import hashlib
#import uuid
(print, rrr, profile) = ut.inject2(__name__, '[preproc_img]', DEBUG=False)


#@profile
def parse_exif(pil_img):
    """ Image EXIF helper

    Cyth::
        cdef:
            Image pil_img
            dict exif_dict
            long lat
            long lon
            long exiftime
    """
    exif_dict = vtexif.get_exif_dict(pil_img)
    # TODO: More tags
    # (mainly the orientation tag)
    lat, lon = vtexif.get_lat_lon(exif_dict)
    orient = vtexif.get_orientation(exif_dict, on_error='warn')
    time = vtexif.get_unixtime(exif_dict)
    return time, lat, lon, orient


def get_standard_ext(gpath):
    """ Returns standardized image extension

    Cyth::
        cdef:
            str gpath
            str ext

    """
    ext = splitext(gpath)[1].lower()
    return '.jpg' if ext == '.jpeg' else ext


@profile
def parse_imageinfo(gpath):
    """ Worker function: gpath must be in UNIX-PATH format!

    Args:
        tup (tuple): a tuple or one argument
            (so the function can be parallelized easily)
            (here it is just gpath, no tuple, sorry for confusion)

    Returns:
        tuple: param_tup -
            if successful returns a tuple of image parameters which are values
            for SQL columns on else returns None

    CommandLine:
        python -m ibeis.algo.preproc.preproc_image --exec-parse_imageinfo

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.algo.preproc.preproc_image import *  # NOQA
        >>> gpath = ('/media/raid/work/lynx/_ibsdb/images/f6c84c6d-55ca-fd02-d0b4-1c7c9c27c894.jpg')
        >>> param_tup = parse_imageinfo(tup)
        >>> result = ('param_tup = %s' % (str(param_tup),))
        >>> print(result)
    """
    # Parse arguments from tuple
    #print('[ginfo] gpath=%r' % gpath)
    # Try to open the image
    from PIL import Image  # NOQA
    from os.path import isabs
    import six
    import tempfile
    if six.PY2:
        import urlparse
        urlsplit = urlparse.urlsplit
    else:
        import urllib
        urlsplit = urllib.parse.urlsplit

    url_protos = ['https://', 'http://']
    s3_proto = ['s3://']
    valid_protos = s3_proto + url_protos

    def isproto(gpath, valid_protos):
        return any(gpath.startswith(proto) for proto in valid_protos)

    def islocal(gpath):
        return not (isabs(gpath) and isproto(gpath, valid_protos))

    with warnings.catch_warnings(record=True) as w:
        try:
            if isproto(gpath, valid_protos):
                suffix = '.%s' % (basename(gpath), )
                temp_file, temp_filepath = tempfile.mkstemp(suffix=suffix)
                args = (gpath, temp_filepath, )
                print('[preproc] Caching remote %s file to temporary file %r' % args)

                if isproto(gpath, s3_proto):
                    s3_dict = ut.s3_str_decode_to_dict(gpath)
                    ut.grab_s3_contents(temp_filepath, **s3_dict)
                if isproto(gpath, url_protos):
                    # Ensure that the Unicode string is properly encoded for web requests
                    uri_ = urlsplit(gpath)
                    uri_path = six.moves.urllib.parse.quote(uri_.path.encode('utf8'))
                    uri_ = uri_._replace(path=uri_path)
                    uri_ = uri_.geturl()
                    six.moves.urllib.request.urlretrieve(uri_, filename=temp_filepath)
                gpath_ = temp_filepath
            else:
                temp_file, temp_filepath = None, None
                gpath_ = gpath

            # Open image with Exif support
            pil_img = Image.open(gpath_, 'r')  # NOQA
            # We cannot use pixel data as libjpeg is not determenistic (even for reads!)
            image_uuid = ut.get_file_uuid(gpath_)  # Read file ]-hash-> guid = gid
        except IOError as ex:
            # ut.embed()<commit_msg>Add images embed<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
import os
from os.path import splitext, basename
import warnings  # NOQA
import vtool.exif as vtexif
import utool as ut
#import numpy as np  # NOQA
#import hashlib
#import uuid
(print, rrr, profile) = ut.inject2(__name__, '[preproc_img]', DEBUG=False)


#@profile
def parse_exif(pil_img):
    """ Image EXIF helper

    Cyth::
        cdef:
            Image pil_img
            dict exif_dict
            long lat
            long lon
            long exiftime
    """
    exif_dict = vtexif.get_exif_dict(pil_img)
    # TODO: More tags
    # (mainly the orientation tag)
    lat, lon = vtexif.get_lat_lon(exif_dict)
    orient = vtexif.get_orientation(exif_dict, on_error='warn')
    time = vtexif.get_unixtime(exif_dict)
    return time, lat, lon, orient


def get_standard_ext(gpath):
    """ Returns standardized image extension

    Cyth::
        cdef:
            str gpath
            str ext

    """
    ext = splitext(gpath)[1].lower()
    return '.jpg' if ext == '.jpeg' else ext


@profile
def parse_imageinfo(gpath):
    """ Worker function: gpath must be in UNIX-PATH format!

    Args:
        tup (tuple): a tuple or one argument
            (so the function can be parallelized easily)
            (here it is just gpath, no tuple, sorry for confusion)

    Returns:
        tuple: param_tup -
            if successful returns a tuple of image parameters which are values
            for SQL columns on else returns None

    CommandLine:
        python -m ibeis.algo.preproc.preproc_image --exec-parse_imageinfo

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.algo.preproc.preproc_image import *  # NOQA
        >>> gpath = ('/media/raid/work/lynx/_ibsdb/images/f6c84c6d-55ca-fd02-d0b4-1c7c9c27c894.jpg')
        >>> param_tup = parse_imageinfo(tup)
        >>> result = ('param_tup = %s' % (str(param_tup),))
        >>> print(result)
    """
    # Parse arguments from tuple
    #print('[ginfo] gpath=%r' % gpath)
    # Try to open the image
    from PIL import Image  # NOQA
    from os.path import isabs
    import six
    import tempfile
    if six.PY2:
        import urlparse
        urlsplit = urlparse.urlsplit
    else:
        import urllib
        urlsplit = urllib.parse.urlsplit

    url_protos = ['https://', 'http://']
    s3_proto = ['s3://']
    valid_protos = s3_proto + url_protos

    def isproto(gpath, valid_protos):
        return any(gpath.startswith(proto) for proto in valid_protos)

    def islocal(gpath):
        return not (isabs(gpath) and isproto(gpath, valid_protos))

    with warnings.catch_warnings(record=True) as w:
        try:
            if isproto(gpath, valid_protos):
                ut.embed()
                suffix = '.%s' % (basename(gpath), )
                temp_file, temp_filepath = tempfile.mkstemp(suffix=suffix)
                args = (gpath, temp_filepath, )
                print('[preproc] Caching remote %s file to temporary file %r' % args)

                if isproto(gpath, s3_proto):
                    s3_dict = ut.s3_str_decode_to_dict(gpath)
                    ut.grab_s3_contents(temp_filepath, **s3_dict)
                if isproto(gpath, url_protos):
                    # Ensure that the Unicode string is properly encoded for web requests
                    uri_ = urlsplit(gpath)
                    uri_path = six.moves.urllib.parse.quote(uri_.path.encode('utf8'))
                    uri_ = uri_._replace(path=uri_path)
                    uri_ = uri_.geturl()
                    six.moves.urllib.request.urlretrieve(uri_, filename=temp_filepath)
                gpath_ = temp_filepath
            else:
                temp_file, temp_filepath = None, None
                gpath_ = gpath

            # Open image with Exif support
            pil_img = Image.open(gpath_, 'r')  # NOQA
            # We cannot use pixel data as libjpeg is not determenistic (even for reads!)
            image_uuid = ut.get_file_uuid(gpath_)  # Read file ]-hash-> guid = gid
        except IOError as ex:
            # ut.embed()
{'fraction_non_alphanumeric': 0.09545016851227732, 'fraction_numerical': 0.007462686567164179, 'mean_word_length': 2.113150992881229, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 41, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2473448', 'n_tokens_mistral': 2700, 'n_tokens_neox': 2610, 'n_words': 690}