<commit_before>from cum import exceptions
import threading
import click
import json
import os
import re
import requests
import sys


class BaseConfig(object):
    def __init__(self):
        self.load()

    def __setattr__(self, name, value):
        """Ensures that changes made after loading with default values
        are written back to disk.
        """
        if hasattr(self, 'persistent_config'):
            self.persistent_config[name] = value
        object.__setattr__(self, name, value)

    @property
    def default_download_directory(self):
        """Returns a platform-specific download directory to use if no download
        directory is specified by the user.
        """
        if sys.platform in ['cygwin', 'win32']:
            return os.path.join(os.environ['USERPROFILE'], 'Downloads')
        else:
            return os.environ['HOME']

    def load(self):
        try:
            f = open(config_path)
        except FileNotFoundError:
            j = {}
        else:
            try:
                j = json.load(f)
            except json.decoder.JSONDecodeError as e:
                f.seek(0, 0)
                raise exceptions.ConfigError(config=f.read(),
                                             cursor=(e.lineno, e.colno),
                                             message='Error reading config: {}'
                                                     .format(e.msg))
            finally:
                f.close()

        self.batoto = BatotoConfig(self, j.get('batoto', {}))<commit_msg>config: Fix config checker for python 3.3/3.4<commit_after>from cum import exceptions
import threading
import click
import json
import os
import re
import requests
import sys


class BaseConfig(object):
    def __init__(self):
        self.load()

    def __setattr__(self, name, value):
        """Ensures that changes made after loading with default values
        are written back to disk.
        """
        if hasattr(self, 'persistent_config'):
            self.persistent_config[name] = value
        object.__setattr__(self, name, value)

    @property
    def default_download_directory(self):
        """Returns a platform-specific download directory to use if no download
        directory is specified by the user.
        """
        if sys.platform in ['cygwin', 'win32']:
            return os.path.join(os.environ['USERPROFILE'], 'Downloads')
        else:
            return os.environ['HOME']

    def load(self):
        try:
            f = open(config_path)
        except FileNotFoundError:
            j = {}
        else:
            try:
                j = json.load(f)
            except ValueError as e:
                f.seek(0, 0)
                cfargs = {}
                if hasattr(json.decoder, 'JSONDecodeError'):
                    cfargs = {'config': f.read(),
                              'cursor': (e.lineno, e.colno),
                              'message': 'Error reading config: {}'
                                         .format(e.msg)}
                else:
                    # Remove this hack when we drop Python 3.4 support
                    msg, pos = str(e).split(':')
                    m = re.match(r'\s*line (\d+) column (\d+).*', pos)
                    cur = (int(m.group(1)), int(m.group(2)))
                    cfargs = {'config': f.read(),
                              'cursor': cur,
                              'message': 'Error reading config: {}'
                                         .format(msg)}
                raise exceptions.ConfigError(**cfargs)
            finally:
                f.close()

        self.batoto = BatotoConfig(self, j.get('batoto', {}))
{'fraction_non_alphanumeric': 0.10369163952225842, 'fraction_numerical': 0.004343105320304018, 'mean_word_length': 1.4881836596893991, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2701821', 'n_tokens_mistral': 955, 'n_tokens_neox': 924, 'n_words': 267}