<commit_before># jsonfile
# (C) <NAME> - UFCG
# gist url: https://gist.github.com/daltonserey/8fc9644cfb9dda7fdfb09a3e006587b3

from __future__ import print_function
from __future__ import unicode_literals

import codecs
import sys
import os
import json

def to_unicode(obj, encoding='utf-8'):
    assert isinstance(obj, basestring), type(obj)
    if isinstance(obj, unicode):
        return obj

    for encoding in ['utf-8', 'latin1']:
        try:
            obj = unicode(obj, encoding)
            return obj
        except UnicodeDecodeError:
            pass

    assert False, "tst: non-recognized encoding"

class CorruptedJsonFile(Exception): pass

class JsonFile(object):

    __instances = {}

    def __new__(cls, filename, exit_on_fail=False, failmsg=None, array2map=None, writable=False):
        filename = os.path.expanduser(filename)
        if filename in JsonFile.__instances:
            return JsonFile.__instances[filename]

        JsonFile.__instances[filename] = object.__new__(cls)
        self = JsonFile.__instances[filename]
        self.filename = filename
        self.writable = writable
        self.isyaml = filename.endswith("yaml") or filename.endswith("yml")
        
        if os.path.exists(filename):
            self.load(failmsg=failmsg, exit_on_fail=exit_on_fail)<commit_msg>Fix error message<commit_after># jsonfile
# (C) <NAME> - UFCG
# gist url: https://gist.github.com/daltonserey/8fc9644cfb9dda7fdfb09a3e006587b3

from __future__ import print_function
from __future__ import unicode_literals

import codecs
import sys
import os
import json

def to_unicode(obj, encoding='utf-8'):
    assert isinstance(obj, basestring), type(obj)
    if isinstance(obj, unicode):
        return obj

    for encoding in ['utf-8', 'latin1']:
        try:
            obj = unicode(obj, encoding)
            return obj
        except UnicodeDecodeError:
            pass

    assert False, "jsonfile: non-recognized encoding"

class CorruptedJsonFile(Exception): pass

class JsonFile(object):

    __instances = {}

    def __new__(cls, filename, exit_on_fail=False, failmsg=None, array2map=None, writable=False):
        filename = os.path.expanduser(filename)
        if filename in JsonFile.__instances:
            return JsonFile.__instances[filename]

        JsonFile.__instances[filename] = object.__new__(cls)
        self = JsonFile.__instances[filename]
        self.filename = filename
        self.writable = writable
        self.isyaml = filename.endswith("yaml") or filename.endswith("yml")
        
        if os.path.exists(filename):
            self.load(failmsg=failmsg, exit_on_fail=exit_on_fail)
{'fraction_non_alphanumeric': 0.0950584685024519, 'fraction_numerical': 0.015843078083741986, 'mean_word_length': 3.137285491419657, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '193641', 'n_tokens_mistral': 843, 'n_tokens_neox': 797, 'n_words': 185}