<commit_before>"""
utils
=====

Low-level functionality NOT intended for users to EVER use.

"""

import json
import os.path
import sys
import threading
import re
import datetime

try:
    import numpy
    _numpy_imported = True
except ImportError:
    _numpy_imported = False

try:
    import pandas
    _pandas_imported = True
except ImportError:
    _pandas_imported = False

try:
    from sage.all import RR, ZZ
    _sage_imported = True
except ImportError:
    _sage_imported = False


### incase people are using threading, we lock file reads
lock = threading.Lock()


### general file setup tools ###

def load_json_dict(filename, *args):
    """Checks if file exists. Returns {} if something fails."""
    data = {}
    if os.path.exists(filename):
        lock.acquire()
        with open(filename, "r") as f:
            try:
                data = json.load(f)
                if not isinstance(data, dict):
                    data = {}
            except:
                data = {}  # TODO: issue a warning and bubble it up
        lock.release()
        if args:
            d = dict()
            for key in args:
                if key in data:
                    d[key] = data[key]
            return d
            # TODO: replace with below if we drop Python 2.6 compatibility
            # return {key: data[key] for key in args if key in data}
    return data


def save_json_dict(filename, json_dict):
    """Save json to file. Error if path DNE, not a dict, or invalid json."""
    if isinstance(json_dict, dict):
        # this will raise a TypeError if something goes wrong
        json_string = json.dumps(json_dict, indent=4)
        lock.acquire()
        with open(filename, "w") as f:
            f.write(json_string)
        lock.release()
    else:
        raise TypeError("json_dict was not a dictionary. not saving.")


def ensure_file_exists(filename):
    """Given a valid filename, make sure it exists (will create if DNE)."""
    if not os.path.exists(filename):
        head, tail = os.path.split(filename)
        ensure_dir_exists(head)
        with open(filename, 'w') as f:
            pass  # just create the file


def ensure_dir_exists(directory):
    """Given a valid directory path, make sure it exists."""
    if dir:
        if not os.path.isdir(directory):
            os.makedirs(directory)


### Custom JSON encoders ###
class NotEncodable(Exception):
    pass


class _plotlyJSONEncoder(json.JSONEncoder):
    def numpyJSONEncoder(self, obj):
        if not _numpy_imported:
            raise NotEncodable

        if obj is numpy.ma.core.masked:
            return float('nan')

        if type(obj).__module__.split('.')[0] == numpy.__name__:
            l = obj.tolist()
            try:
                return self.datetimeJSONEncoder(l)
            except NotEncodable:
                return l
        else:
            raise NotEncodable

    def datetimeJSONEncoder(self, obj):
        """
        if datetime or iterable of datetimes,
        convert to a string that plotly understands
        format as %Y-%m-%d %H:%M:%S.%f,
                  %Y-%m-%d %H:%M:%S, or
                  %Y-%m-%d
        depending on what non-zero resolution was provided
        """

        if _pandas_imported and obj is pandas.NaT:
            return None

        if isinstance(obj, (datetime.datetime, datetime.date)):
            if obj.microsecond:
                return obj.strftime('%Y-%m-%d %H:%M:%S.%f')
            elif any((obj.second, obj.minute, obj.hour)):
                return obj.strftime('%Y-%m-%d %H:%M:%S')
            else:
                return obj.strftime('%Y-%m-%d')
        elif isinstance(obj[0], (datetime.datetime, datetime.date)):
            return [o.strftime(
                    '%Y-%m-%d %H:%M:%S.%f') if o.microsecond else
                    o.strftime('%Y-%m-%d %H:%M:%S') if any((o.second, o.minute, o.hour)) else
                    o.strftime('%Y-%m-%d')
                    for o in obj]
        else:
            raise NotEncodable

    def pandasJSONEncoder(self, obj):<commit_msg>json cls encoders recurse, so no need to index<commit_after>"""
utils
=====

Low-level functionality NOT intended for users to EVER use.

"""

import json
import os.path
import sys
import threading
import re
import datetime

try:
    import numpy
    _numpy_imported = True
except ImportError:
    _numpy_imported = False

try:
    import pandas
    _pandas_imported = True
except ImportError:
    _pandas_imported = False

try:
    from sage.all import RR, ZZ
    _sage_imported = True
except ImportError:
    _sage_imported = False


### incase people are using threading, we lock file reads
lock = threading.Lock()


### general file setup tools ###

def load_json_dict(filename, *args):
    """Checks if file exists. Returns {} if something fails."""
    data = {}
    if os.path.exists(filename):
        lock.acquire()
        with open(filename, "r") as f:
            try:
                data = json.load(f)
                if not isinstance(data, dict):
                    data = {}
            except:
                data = {}  # TODO: issue a warning and bubble it up
        lock.release()
        if args:
            d = dict()
            for key in args:
                if key in data:
                    d[key] = data[key]
            return d
            # TODO: replace with below if we drop Python 2.6 compatibility
            # return {key: data[key] for key in args if key in data}
    return data


def save_json_dict(filename, json_dict):
    """Save json to file. Error if path DNE, not a dict, or invalid json."""
    if isinstance(json_dict, dict):
        # this will raise a TypeError if something goes wrong
        json_string = json.dumps(json_dict, indent=4)
        lock.acquire()
        with open(filename, "w") as f:
            f.write(json_string)
        lock.release()
    else:
        raise TypeError("json_dict was not a dictionary. not saving.")


def ensure_file_exists(filename):
    """Given a valid filename, make sure it exists (will create if DNE)."""
    if not os.path.exists(filename):
        head, tail = os.path.split(filename)
        ensure_dir_exists(head)
        with open(filename, 'w') as f:
            pass  # just create the file


def ensure_dir_exists(directory):
    """Given a valid directory path, make sure it exists."""
    if dir:
        if not os.path.isdir(directory):
            os.makedirs(directory)


### Custom JSON encoders ###
class NotEncodable(Exception):
    pass


class _plotlyJSONEncoder(json.JSONEncoder):
    def numpyJSONEncoder(self, obj):
        if not _numpy_imported:
            raise NotEncodable

        if obj is numpy.ma.core.masked:
            return float('nan')

        if type(obj).__module__.split('.')[0] == numpy.__name__:
            return obj.tolist()

        else:
            raise NotEncodable

    def datetimeJSONEncoder(self, obj):
        """
        if datetime or iterable of datetimes,
        convert to a string that plotly understands
        format as %Y-%m-%d %H:%M:%S.%f,
                  %Y-%m-%d %H:%M:%S, or
                  %Y-%m-%d
        depending on what non-zero resolution was provided
        """

        if _pandas_imported and obj is pandas.NaT:
            return None

        if isinstance(obj, (datetime.datetime, datetime.date)):
            if obj.microsecond:
                return obj.strftime('%Y-%m-%d %H:%M:%S.%f')
            elif any((obj.second, obj.minute, obj.hour)):
                return obj.strftime('%Y-%m-%d %H:%M:%S')
            else:
                return obj.strftime('%Y-%m-%d')
        else:
            raise NotEncodable

    def pandasJSONEncoder(self, obj):
{'fraction_non_alphanumeric': 0.10643532306098666, 'fraction_numerical': 0.0011653502524925548, 'mean_word_length': 1.975346687211094, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4690406', 'n_tokens_mistral': 2312, 'n_tokens_neox': 2229, 'n_words': 711}