<commit_before># jsonfile
# (C) <NAME> - UFCG

"""
# usage

Use it as a simple dict with a save method that saves the data to the file. If
the file exists it will be read at instantiation. If you create more than one
object for the same filename, the same object will be returned by the
constructor (it is a singleton). See the example below.

```
from jsonfile import JsonFile

f = JsonFile('/Users/dalton/somefile.json', writable=True)
f['name'] = 'dalton'
f['data'] = [1, 2, 3]
f.save()
f2 = JsonFile('/Users/dalton/somefile.json')
f2 is f # True
```
"""

from __future__ import print_function
from __future__ import unicode_literals

import codecs
import sys
import os
import json

def to_unicode(obj, encoding='utf-8'):
    assert isinstance(obj, basestring), type(obj)
    if isinstance(obj, unicode):
        return obj

    for encoding in ['utf-8', 'latin1']:
        try:
            obj = unicode(obj, encoding)
            return obj
        except UnicodeDecodeError:
            pass

    assert False, "jsonfile: non-recognized encoding"

class CorruptedJsonFile(Exception): pass

class JsonFile(object):

    __instances = {}

    def __new__(cls, filename, exit_on_fail=False, failmsg=None, array2map=None, writable=False):
        filename = os.path.expanduser(filename)
        if filename in JsonFile.__instances:
            return JsonFile.__instances[filename]

        JsonFile.__instances[filename] = object.__new__(cls)
        self = JsonFile.__instances[filename]
        self.filename = filename
        self.writable = writable
        self.isyaml = filename.endswith("yaml") or filename.endswith("yml")
        
        if os.path.exists(filename):
            self.load(failmsg=failmsg, exit_on_fail=exit_on_fail)
            if array2map and type(self.data) is list:
                self.data = { array2map: self.data }

        else:
            self.data = {}

        return self


    def __setitem__(self, key, value):
        self.data[key] = value


    def __getitem__(self, key):
        return self.data[key]


    def __contains__(self, key):
        return key in self.data


    def pop(self, key):
        self.data.pop(key, None)


    def setdefault(self, key, value):
        return self.data.setdefault(key, value)


    def load(self, exit_on_fail=False, failmsg=None):
        if not os.path.exists(self.filename):
            self.data = {}
            return

        # actually read data from file system
        try:
            if self.isyaml:
                import yaml
                with codecs.open(self.filename, mode='r', encoding='utf-8') as f:
                    self.data = yaml.load(to_unicode(f.read()))
                    if self.data is None:
                        raise ValueError()
            else:    
                with codecs.open(self.filename, mode='r', encoding='utf-8') as f:
                    self.data = json.loads(to_unicode(f.read()))

        except ValueError:
            if exit_on_fail or failmsg:
                failmsg = failmsg or "jsonfile: %s is corrupted" % self.filename
                print(failmsg, file=sys.stderr)
                sys.exit()

            raise CorruptedJsonFile("corrupted json file")


    def save(self):
        assert self.writable, "jsonfile: cannot save a non writable JsonFile"
        with codecs.open(self.filename, mode="w", encoding='utf-8') as f:
            f.write(json.dumps(
                self.data,
                indent=2,
                separators=(',', ': ')
            ))


    def get(self, key, default=None):
        return self.data.get(key, default)<commit_msg>Minor rewrite of reading json and yaml files<commit_after># jsonfile
# (C) <NAME> - UFCG

"""
# usage

Use it as a simple dict with a save method that saves the data to the file. If
the file exists it will be read at instantiation. If you create more than one
object for the same filename, the same object will be returned by the
constructor (it is a singleton). See the example below.

```
from jsonfile import JsonFile

f = JsonFile('/Users/dalton/somefile.json', writable=True)
f['name'] = 'dalton'
f['data'] = [1, 2, 3]
f.save()
f2 = JsonFile('/Users/dalton/somefile.json')
f2 is f # True
```
"""

from __future__ import print_function
from __future__ import unicode_literals

import codecs
import sys
import os
import json

def to_unicode(obj, encoding='utf-8'):
    assert isinstance(obj, basestring), type(obj)
    if isinstance(obj, unicode):
        return obj

    for encoding in ['utf-8', 'latin1']:
        try:
            obj = unicode(obj, encoding)
            return obj
        except UnicodeDecodeError:
            pass

    assert False, "jsonfile: non-recognized encoding"

class CorruptedJsonFile(Exception): pass

DEFAULT_FAIL_MESSAGE = "file is corrupted"

class JsonFile(object):

    __instances = {}

    def __new__(cls, filename, exit_on_fail=False, failmsg=None, array2map=None, writable=False):
        filename = os.path.expanduser(filename)
        if filename in JsonFile.__instances:
            return JsonFile.__instances[filename]

        JsonFile.__instances[filename] = object.__new__(cls)
        self = JsonFile.__instances[filename]
        self.filename = filename
        self.writable = writable
        self.isyaml = filename.endswith("yaml") or filename.endswith("yml")
        
        if os.path.exists(filename):
            self.load(failmsg=failmsg, exit_on_fail=exit_on_fail)
            if array2map and type(self.data) is list:
                self.data = { array2map: self.data }

        else:
            self.data = {}

        return self


    def __setitem__(self, key, value):
        self.data[key] = value


    def __getitem__(self, key):
        return self.data[key]


    def __contains__(self, key):
        return key in self.data


    def pop(self, key):
        self.data.pop(key, None)


    def setdefault(self, key, value):
        return self.data.setdefault(key, value)


    def load(self, exit_on_fail=False, failmsg=None):
        if not os.path.exists(self.filename):
            self.data = {}
            return

        # actually read data from file system
        if self.isyaml:
            import yaml
            try:
                with codecs.open(self.filename, mode='r', encoding='utf-8') as f:
                    self.data = yaml.load(to_unicode(f.read()))
                    if self.data is None:
                        raise ValueError()

            except (ValueError, yaml.scanner.ScannerError) as e:
                if exit_on_fail or failmsg:
                    print(failmsg or DEFAULT_FAIL_MESSAGE, file=sys.stderr)
                    sys.exit()

                raise CorruptedJsonFile("corrupted specification file")

        else: # assume is json
            try:
                with codecs.open(self.filename, mode='r', encoding='utf-8') as f:
                    self.data = json.loads(to_unicode(f.read()))

            except ValueError as e:
                if exit_on_fail or failmsg:
                    print(failmsg or DEFAULT_FAIL_MESSAGE, file=sys.stderr)
                    sys.exit(1)

                raise CorruptedJsonFile("corrupted specification file")



    def save(self):
        assert self.writable, "jsonfile: cannot save a non writable JsonFile"
        with codecs.open(self.filename, mode="w", encoding='utf-8') as f:
            f.write(json.dumps(
                self.data,
                indent=2,
                separators=(',', ': ')
            ))


    def get(self, key, default=None):
        return self.data.get(key, default)
{'fraction_non_alphanumeric': 0.10211591536338546, 'fraction_numerical': 0.004074122749375739, 'mean_word_length': 2.1708333333333334, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '298502', 'n_tokens_mistral': 2280, 'n_tokens_neox': 2157, 'n_words': 630}