<commit_before># -*- coding: utf-8 -*-

"""Utility functions."""


#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import base64
import json
import os.path as op
import os
import sys
import subprocess
from inspect import getargspec

import numpy as np

from ..ext.six import string_types, exec_
from ..ext.six.moves import builtins, cPickle


#------------------------------------------------------------------------------
# Pickle utility functions
#------------------------------------------------------------------------------

def _load_pickle(path):
    path = op.realpath(op.expanduser(path))
    assert op.exists(path)
    with open(path, 'rb') as f:
        return cPickle.load(f)


def _save_pickle(path, data):
    path = op.realpath(op.expanduser(path))
    with open(path, 'wb') as f:
        cPickle.dump(data, f, protocol=2)


#------------------------------------------------------------------------------
# JSON utility functions
#------------------------------------------------------------------------------

class _NumpyEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.ndarray):
            data_b64 = base64.b64encode(obj.data).decode('utf8')
            return dict(__ndarray__=data_b64,
                        dtype=str(obj.dtype),
                        shape=obj.shape)
        return super(_NumpyEncoder, self).default(obj)


def _json_numpy_obj_hook(d):
    if isinstance(d, dict) and '__ndarray__' in d:
        data = base64.b64decode(d['__ndarray__'])
        return np.frombuffer(data, d['dtype']).reshape(d['shape'])
    return d


def _intify_keys(d):
    assert isinstance(d, dict)
    out = {}
    for k, v in d.items():
        if isinstance(k, string_types) and k.isdigit():
            k = int(k)
        out[k] = v
    return out


def _load_json(path):
    path = op.realpath(op.expanduser(path))
    assert op.exists(path)
    with open(path, 'r') as f:
        out = json.load(f, object_hook=_json_numpy_obj_hook)
        return _intify_keys(out)


def _save_json(path, data):
    assert isinstance(data, dict)
    data = _intify_keys(data)
    path = op.realpath(op.expanduser(path))
    with open(path, 'w') as f:<commit_msg>WIP: fixing integer key problem in JSON serialization.<commit_after># -*- coding: utf-8 -*-

"""Utility functions."""


#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import base64
import json
import os.path as op
import os
import sys
import subprocess
from inspect import getargspec

import numpy as np

from ._types import _is_integer
from ..ext.six import string_types, exec_
from ..ext.six.moves import builtins, cPickle


#------------------------------------------------------------------------------
# Pickle utility functions
#------------------------------------------------------------------------------

def _load_pickle(path):
    path = op.realpath(op.expanduser(path))
    assert op.exists(path)
    with open(path, 'rb') as f:
        return cPickle.load(f)


def _save_pickle(path, data):
    path = op.realpath(op.expanduser(path))
    with open(path, 'wb') as f:
        cPickle.dump(data, f, protocol=2)


#------------------------------------------------------------------------------
# JSON utility functions
#------------------------------------------------------------------------------

class _NumpyEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.ndarray):
            data_b64 = base64.b64encode(obj.data).decode('utf8')
            return dict(__ndarray__=data_b64,
                        dtype=str(obj.dtype),
                        shape=obj.shape)
        return super(_NumpyEncoder, self).default(obj)


def _json_numpy_obj_hook(d):
    if isinstance(d, dict) and '__ndarray__' in d:
        data = base64.b64decode(d['__ndarray__'])
        return np.frombuffer(data, d['dtype']).reshape(d['shape'])
    return d


def _intify_keys(d):
    assert isinstance(d, dict)
    out = {}
    for k, v in d.items():
        if isinstance(k, string_types) and k.isdigit():
            k = int(k)
        out[k] = v
    return out


def _stringify_keys(d):
    assert isinstance(d, dict)
    out = {}
    for k, v in d.items():
        if _is_integer(k):
            k = str(k)
        out[k] = v
    return out


def _load_json(path):
    path = op.realpath(op.expanduser(path))
    assert op.exists(path)
    with open(path, 'r') as f:
        out = json.load(f, object_hook=_json_numpy_obj_hook)
        return _intify_keys(out)


def _save_json(path, data):
    assert isinstance(data, dict)
    data = _stringify_keys(data)
    path = op.realpath(op.expanduser(path))
    with open(path, 'w') as f:
{'fraction_non_alphanumeric': 0.2901322482197355, 'fraction_numerical': 0.006917599186164801, 'mean_word_length': 1.4828282828282828, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5856779', 'n_tokens_mistral': 1414, 'n_tokens_neox': 1382, 'n_words': 341}