<commit_before>from __future__ import division

import datetime
import exifread
import logging
import xmltodict as x2d

from six import string_types

from opensfm.sensors import sensor_data
from opensfm import types


logger = logging.getLogger(__name__)

inch_in_mm = 25.4
cm_in_mm = 10
um_in_mm = 0.001


def eval_frac(value):
    try:
        return float(value.num) / float(value.den)
    except ZeroDivisionError:
        return None


def gps_to_decimal(values, reference):
    sign = 1 if reference in 'NE' else -1
    degrees = eval_frac(values[0])
    minutes = eval_frac(values[1])
    seconds = eval_frac(values[2])
    return sign * (degrees + minutes / 60 + seconds / 3600)


def get_tag_as_float(tags, key, index=0):
    if key in tags:
        val = tags[key].values[index]
        if isinstance(val, exifread.utils.Ratio):
            ret_val = eval_frac(val)
            if ret_val is None:
                logger.error(
                    'The rational "{2}" of tag "{0:s}" at index {1:d} c'
                    'aused a division by zero error'.format(
                        key, index, val))
            return ret_val
        else:
            return float(val)
    else:
        return None


def compute_focal(focal_35, focal, sensor_width, sensor_string):
    if focal_35 is not None and focal_35 > 0:
        focal_ratio = focal_35 / 36.0  # 35mm film produces 36x24mm pictures.
    else:
        if not sensor_width:
            sensor_width = sensor_data.get(sensor_string, None)
        if sensor_width and focal:
            focal_ratio = focal / sensor_width
            focal_35 = 36.0 * focal_ratio
        else:
            focal_35 = 0
            focal_ratio = 0
    return focal_35, focal_ratio


def sensor_string(make, model):
    if make != 'unknown':
        # remove duplicate 'make' information in 'model'
        model = model.replace(make, '')
    return (make.strip() + ' ' + model.strip()).strip().lower()


def camera_id(exif):
    return camera_id_(exif['make'], exif['model'],
                      exif['width'], exif['height'],
                      exif['projection_type'], exif['focal_ratio'])


def camera_id_(make, model, width, height, projection_type, focal):
    if make != 'unknown':
        # remove duplicate 'make' information in 'model'
        model = model.replace(make, '')
    return ' '.join([
        'v2',
        make.strip(),
        model.strip(),
        str(int(width)),
        str(int(height)),
        projection_type,
        str(focal)[:6],
    ]).lower()


def extract_exif_from_file(fileobj):
    if isinstance(fileobj, string_types):
        with open(fileobj) as f:
            exif_data = EXIF(f)
    else:
        exif_data = EXIF(fileobj)

    d = exif_data.extract_exif()
    return d


def get_xmp(fileobj):
    '''Extracts XMP metadata from and image fileobj
    '''
    img_str = str(fileobj.read())
    xmp_start = img_str.find('<x:xmpmeta')
    xmp_end = img_str.find('</x:xmpmeta')

    if xmp_start < xmp_end:
        xmp_str = img_str[xmp_start:xmp_end + 12]
        xdict = x2d.parse(xmp_str)
        xdict = xdict.get('x:xmpmeta', {})
        xdict = xdict.get('rdf:RDF', {})
        xdict = xdict.get('rdf:Description', {})
        if isinstance(xdict, list):
            return xdict
        else:
            return [xdict]
    else:
        return []


def get_gpano_from_xmp(xmp):
    for i in xmp:
        for k in i:<commit_msg>feat: handle escaped xmp strings<commit_after>from __future__ import division

import datetime
import exifread
import logging
import xmltodict as x2d
from codecs import encode, decode

from six import string_types

from opensfm.sensors import sensor_data
from opensfm import types


logger = logging.getLogger(__name__)

inch_in_mm = 25.4
cm_in_mm = 10
um_in_mm = 0.001


def eval_frac(value):
    try:
        return float(value.num) / float(value.den)
    except ZeroDivisionError:
        return None


def gps_to_decimal(values, reference):
    sign = 1 if reference in 'NE' else -1
    degrees = eval_frac(values[0])
    minutes = eval_frac(values[1])
    seconds = eval_frac(values[2])
    return sign * (degrees + minutes / 60 + seconds / 3600)


def get_tag_as_float(tags, key, index=0):
    if key in tags:
        val = tags[key].values[index]
        if isinstance(val, exifread.utils.Ratio):
            ret_val = eval_frac(val)
            if ret_val is None:
                logger.error(
                    'The rational "{2}" of tag "{0:s}" at index {1:d} c'
                    'aused a division by zero error'.format(
                        key, index, val))
            return ret_val
        else:
            return float(val)
    else:
        return None


def compute_focal(focal_35, focal, sensor_width, sensor_string):
    if focal_35 is not None and focal_35 > 0:
        focal_ratio = focal_35 / 36.0  # 35mm film produces 36x24mm pictures.
    else:
        if not sensor_width:
            sensor_width = sensor_data.get(sensor_string, None)
        if sensor_width and focal:
            focal_ratio = focal / sensor_width
            focal_35 = 36.0 * focal_ratio
        else:
            focal_35 = 0
            focal_ratio = 0
    return focal_35, focal_ratio


def sensor_string(make, model):
    if make != 'unknown':
        # remove duplicate 'make' information in 'model'
        model = model.replace(make, '')
    return (make.strip() + ' ' + model.strip()).strip().lower()


def camera_id(exif):
    return camera_id_(exif['make'], exif['model'],
                      exif['width'], exif['height'],
                      exif['projection_type'], exif['focal_ratio'])


def camera_id_(make, model, width, height, projection_type, focal):
    if make != 'unknown':
        # remove duplicate 'make' information in 'model'
        model = model.replace(make, '')
    return ' '.join([
        'v2',
        make.strip(),
        model.strip(),
        str(int(width)),
        str(int(height)),
        projection_type,
        str(focal)[:6],
    ]).lower()


def extract_exif_from_file(fileobj):
    if isinstance(fileobj, string_types):
        with open(fileobj) as f:
            exif_data = EXIF(f)
    else:
        exif_data = EXIF(fileobj)

    d = exif_data.extract_exif()
    return d


def unescape_string(s):
    return decode(encode(s, 'latin-1', 'backslashreplace'), 'unicode-escape')


def parse_xmp_string(xmp_str):
    for _ in range(2):
        try:
            return x2d.parse(xmp_str)
        except:
            xmp_str = unescape_string(xmp_str)
    return None


def get_xmp(fileobj):
    '''Extracts XMP metadata from and image fileobj
    '''
    img_str = str(fileobj.read())
    xmp_start = img_str.find('<x:xmpmeta')
    xmp_end = img_str.find('</x:xmpmeta')

    if xmp_start < xmp_end:
        xmp_str = img_str[xmp_start:xmp_end + 12]
        xdict = parse_xmp_string(xmp_str)
        if xdict is None:
            return []
        xdict = xdict.get('x:xmpmeta', {})
        xdict = xdict.get('rdf:RDF', {})
        xdict = xdict.get('rdf:Description', {})
        if isinstance(xdict, list):
            return xdict
        else:
            return [xdict]
    else:
        return []


def get_gpano_from_xmp(xmp):
    for i in xmp:
        for k in i:
{'fraction_non_alphanumeric': 0.10979636763896533, 'fraction_numerical': 0.01651073197578426, 'mean_word_length': 2.2743243243243243, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5167695', 'n_tokens_mistral': 2465, 'n_tokens_neox': 2342, 'n_words': 597}