<commit_before>from datetime import datetime

import six
import json
import pytz
import logging

logger = logging.getLogger()


def timestamp():
    return pytz.utc.localize(datetime.utcnow()).isoformat()


def copy_to_unicode(element):
    """ used to transform the lxml version of unicode to a
    standard version of unicode that can be pickalable -
    necessary for linting """

    if isinstance(element, dict):
        for key, val in element.items():
            element[key] = copy_to_unicode(val)
    elif isinstance(element, list):
        for idx, item in enumerate(element):
            element[idx] = copy_to_unicode(item)
    else:
        try:
            # A dirty way to convert to unicode in python 2 + 3.3+
            element = u''.join(element)
        except TypeError:
            pass
    return element


def stamp_from_raw(raw_doc, **kwargs):
    kwargs['normalizeFinished'] = timestamp()
    stamps = raw_doc['timestamps']
    stamps.update(kwargs)
    return stamps


def format_date_with_slashes(date):
    return date.strftime('%m/%d/%Y')


def json_without_bytes(jobj):
    """
        An ugly hack.

        Before we treat a structure as JSON, ensure that bytes are decoded to str.
    """
    # Create a JSON-compatible copy of the attributes for validation
    jobj = jobj.copy()
    for k, v in jobj.items():
        if isinstance(v, six.binary_type):
            jobj[k] = v.decode('utf8')
    return jobj


def doc_to_normed_dict(doc):
    # make the new dict actually contain real items
    normed = {}
    for key, value in dict(doc).items():
        try:
            normed[key] = json.loads(value)
        except (ValueError, TypeError):
            normed[key] = value

    # Remove empty values and strip down to only normalized fields
    do_not_include = ['docID', 'doc', 'filetype', 'timestamps', 'source']
    for key in list(normed.keys()):
        if not normed[key]:
            del normed[key]

    for key in list(normed.keys()):
        if key in do_not_include:
            del normed[key]

    if normed.get('versions'):
        normed['versions'] = list(map(str, normed['versions']))

    # No datetime means the document wasn't normalized (probably wasn't on the approved list)
    if normed.get('providerUpdatedDateTime'):
        normed['providerUpdatedDateTime'] = normed['providerUpdatedDateTime'].isoformat()

    return normed<commit_msg>Remove util function only used in migrations<commit_after>from datetime import datetime

import six
import json
import pytz
import logging

logger = logging.getLogger()


def timestamp():
    return pytz.utc.localize(datetime.utcnow()).isoformat()


def copy_to_unicode(element):
    """ used to transform the lxml version of unicode to a
    standard version of unicode that can be pickalable -
    necessary for linting """

    if isinstance(element, dict):
        for key, val in element.items():
            element[key] = copy_to_unicode(val)
    elif isinstance(element, list):
        for idx, item in enumerate(element):
            element[idx] = copy_to_unicode(item)
    else:
        try:
            # A dirty way to convert to unicode in python 2 + 3.3+
            element = u''.join(element)
        except TypeError:
            pass
    return element


def stamp_from_raw(raw_doc, **kwargs):
    kwargs['normalizeFinished'] = timestamp()
    stamps = raw_doc['timestamps']
    stamps.update(kwargs)
    return stamps


def format_date_with_slashes(date):
    return date.strftime('%m/%d/%Y')


def json_without_bytes(jobj):
    """
        An ugly hack.

        Before we treat a structure as JSON, ensure that bytes are decoded to str.
    """
    # Create a JSON-compatible copy of the attributes for validation
    jobj = jobj.copy()
    for k, v in jobj.items():
        if isinstance(v, six.binary_type):
            jobj[k] = v.decode('utf8')
    return jobj
{'fraction_non_alphanumeric': 0.1000257135510414, 'fraction_numerical': 0.0020570840833119054, 'mean_word_length': 2.762088974854932, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2587983', 'n_tokens_mistral': 1169, 'n_tokens_neox': 1123, 'n_words': 362}