<commit_before>"""JSON data files."""

import datetime
import json
import logging

import dateutil.parser

log = logging.getLogger(__name__)


def write_json(fp, obj):
    """Write JSON file."""
    json.dump(obj, fp, cls=MyJSONEncoder, ensure_ascii=False, indent=4,
              sort_keys=True)


def read_json(fp):
    """Read JSON file."""
    return json.load(fp, cls=MyJSONDecoder)


class MyJSONEncoder(json.JSONEncoder):
    """Custom JSON encoder."""
    def __init__(self, *args, **kwargs):
        kwargs.update(default=self.default_decode)
        super().__init__(*args, **kwargs)

    def default_decode(self, o):
        """Return a serializable object for custom objects."""
        if isinstance(o, set):
            return list(o)
        # if hasattr(o, '_asdict'):
        #     d = o._asdict()
        #     d['__type__'] = type(o).__name__
        #     return d
        if hasattr(o, 'as_json'):
            return o.as_json()
        # if isinstance(o, datetime.datetime):<commit_msg>use indent=2<commit_after>"""JSON data files."""

import datetime
import json
import logging

import dateutil.parser

log = logging.getLogger(__name__)


def write_json(fp, obj):
    """Write JSON file."""
    json.dump(obj, fp, cls=MyJSONEncoder, ensure_ascii=False, indent=2,
              sort_keys=True)


def read_json(fp):
    """Read JSON file."""
    return json.load(fp, cls=MyJSONDecoder)


class MyJSONEncoder(json.JSONEncoder):
    """Custom JSON encoder."""
    def __init__(self, *args, **kwargs):
        kwargs.update(default=self.default_decode)
        super().__init__(*args, **kwargs)

    def default_decode(self, o):
        """Return a serializable object for custom objects."""
        if isinstance(o, set):
            return list(o)
        # if hasattr(o, '_asdict'):
        #     d = o._asdict()
        #     d['__type__'] = type(o).__name__
        #     return d
        if hasattr(o, 'as_json'):
            return o.as_json()
        # if isinstance(o, datetime.datetime):
{'fraction_non_alphanumeric': 0.13945409429280398, 'fraction_numerical': 0.001488833746898263, 'mean_word_length': 2.7333333333333334, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5139206', 'n_tokens_mistral': 626, 'n_tokens_neox': 617, 'n_words': 154}