<commit_before># -*- coding: utf-8 -*-
import re
import glob
import os
import imp
from collections import OrderedDict
from string import Template

MAX_NESTING_LEVELS = 4

TIME_FORMAT = '%Y-%m-%d %H:%M:%S'
TZTIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
TIME_MODULES = ['time', 'tztime']

I3STATUS_MODULES = [
    'battery', 'cpu_temperature', 'disk', 'ethernet', 'path_exists',
    'run_watch', 'tztime', 'volume', 'wireless'
]
I3S_SINGLE_NAMES = ['cpu_usage', 'ddate', 'ipv6', 'load', 'time']
ERROR_CONFIG = '''
general {colors = true interval = 60}

order += "static_string py3status"
order += "tztime local"
order += "group error"

static_string py3status {format = "py3status"}
tztime local {format = "%c"}
group error{
    button_next = 1
    button_prev = 0
    fixed_width = False
    format = "{output}"
    static_string error_min {format = "CONFIG ERROR" color = "#FF0000"}
    static_string error {format = "$error" color = "#FF0000"}
}
'''

GENERAL_DEFAULTS = {
    'color_bad': '#FF0000',
    'color_degraded': '#FFFF00',
    'color_good': '#00FF00',
    'color_separator': '#333333',
    'colors': False,
    'interval': 5,
    'output_format': 'i3bar',
}


class ParseException(Exception):
    def __init__(self, error, line, line_no, position, token):
        self.error = error
        self.line = line
        self.line_no = line_no
        self.position = position
        self.token = token

    def one_line(self):
        return 'CONFIG ERROR: {} saw `{}` at line {} position {}'.format(
            self.error, self.token, self.line_no, self.position)

    def __str__(self):
        marker = ' ' * (self.position - 1) + '^'
        return '{}\n\nsaw `{}` at line {} position {}\n\n{}\n{}'.format(
            self.error, self.token, self.line_no, self.position, self.line,
            marker)


class ModuleDefinition(OrderedDict):
    '''Module definition in dict form'''
    pass


class ConfigParser:

    TOKENS = [
        '#.*$'  # comments
        '|(?P<operator>[()[\]{},:]|\+?=)'  # operators
        '|(?P<literal>'
        r'("(?:[^"\\]|\\.)*")'  # double quoted string
        r"|('(?:[^'\\]|\\.)*')"  # single quoted string
        '|([a-z_][a-z0-9_]*)'  # token
        '|(-?\d+\.\d*)|(-?\.\d+)'  # float
        '|(-?\d+)'  # int
        ')'
        r'|(?P<newline>\n)'  # newline
        '|(?P<unknown>\S+)'  # unknown token
    ]

    def __init__(self, config):
        self.tokenize(config)
        self.config = {}
        self.level = 0
        self.module_level = 0
        self.current_module = []
        self.current_token = 0
        self.line = 0
        self.raw = config.split('\n')
        self.modules()

    def modules(self):
        modules = []
        root = os.path.dirname(os.path.realpath(__file__))
        module_path = os.path.join(root, 'modules', '*.py')
        for file in glob.glob(module_path):
            modules.append(os.path.basename(file)[:-3])
        self.module_names = modules + I3S_SINGLE_NAMES + I3STATUS_MODULES
        self.container_modules = []

    def check_child_friendly(self, name):
        name = name.split()[0]
        if name in self.container_modules:
            return
        root = os.path.dirname(os.path.realpath(__file__))
        module_path = os.path.join(root, 'modules')
        info = imp.find_module(name, [module_path])
        if not info:
            return
        (file, pathname, description) = info
        py_mod = imp.load_module(name, file, pathname, description)
        if hasattr(py_mod, 'Py3status'):
            if hasattr(py_mod.Py3status, 'Meta'):
                if hasattr(py_mod.Py3status.Meta, 'container'):
                    self.container_modules.append(name)
                    del py_mod
                    return
        del py_mod
        self.error('Module `{}` cannot contain others'.format(name))

    def check_module_name(self, name, offset=0):
        if name in ['general']:
            return
        split_name = name.split()
        if split_name[0] not in self.module_names:
            self.current_token -= len(split_name) - offset
            self.error('Unknown module')
        if len(split_name) > 1 and split_name[0] in I3S_SINGLE_NAMES:
            self.current_token -= len(split_name) - 1 - offset
            self.error('Invalid name cannot have 2 tokens')
        if len(split_name) > 2:
            self.current_token -= len(split_name) - 2 - offset
            self.error('Invalid name cannot have more than 2 tokens')

    def error(self, msg, previous=False):
        '''
        Raise a ParseException.
        We provide information to help locate the error in the config to allow
        easy config debugging for users.  previous indicates that the error
        actually occurred at the end of the previous line.
        '''
        token = self.tokens[self.current_token - 1]<commit_msg>deal with broken modules<commit_after># -*- coding: utf-8 -*-
import re
import glob
import os
import imp
from collections import OrderedDict
from string import Template

MAX_NESTING_LEVELS = 4

TIME_FORMAT = '%Y-%m-%d %H:%M:%S'
TZTIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
TIME_MODULES = ['time', 'tztime']

I3STATUS_MODULES = [
    'battery', 'cpu_temperature', 'disk', 'ethernet', 'path_exists',
    'run_watch', 'tztime', 'volume', 'wireless'
]
I3S_SINGLE_NAMES = ['cpu_usage', 'ddate', 'ipv6', 'load', 'time']
ERROR_CONFIG = '''
general {colors = true interval = 60}

order += "static_string py3status"
order += "tztime local"
order += "group error"

static_string py3status {format = "py3status"}
tztime local {format = "%c"}
group error{
    button_next = 1
    button_prev = 0
    fixed_width = False
    format = "{output}"
    static_string error_min {format = "CONFIG ERROR" color = "#FF0000"}
    static_string error {format = "$error" color = "#FF0000"}
}
'''

GENERAL_DEFAULTS = {
    'color_bad': '#FF0000',
    'color_degraded': '#FFFF00',
    'color_good': '#00FF00',
    'color_separator': '#333333',
    'colors': False,
    'interval': 5,
    'output_format': 'i3bar',
}


class ParseException(Exception):
    def __init__(self, error, line, line_no, position, token):
        self.error = error
        self.line = line
        self.line_no = line_no
        self.position = position
        self.token = token

    def one_line(self):
        return 'CONFIG ERROR: {} saw `{}` at line {} position {}'.format(
            self.error, self.token, self.line_no, self.position)

    def __str__(self):
        marker = ' ' * (self.position - 1) + '^'
        return '{}\n\nsaw `{}` at line {} position {}\n\n{}\n{}'.format(
            self.error, self.token, self.line_no, self.position, self.line,
            marker)


class ModuleDefinition(OrderedDict):
    '''Module definition in dict form'''
    pass


class ConfigParser:

    TOKENS = [
        '#.*$'  # comments
        '|(?P<operator>[()[\]{},:]|\+?=)'  # operators
        '|(?P<literal>'
        r'("(?:[^"\\]|\\.)*")'  # double quoted string
        r"|('(?:[^'\\]|\\.)*')"  # single quoted string
        '|([a-z_][a-z0-9_]*)'  # token
        '|(-?\d+\.\d*)|(-?\.\d+)'  # float
        '|(-?\d+)'  # int
        ')'
        r'|(?P<newline>\n)'  # newline
        '|(?P<unknown>\S+)'  # unknown token
    ]

    def __init__(self, config):
        self.tokenize(config)
        self.config = {}
        self.level = 0
        self.module_level = 0
        self.current_module = []
        self.current_token = 0
        self.line = 0
        self.raw = config.split('\n')
        self.modules()

    def modules(self):
        modules = []
        root = os.path.dirname(os.path.realpath(__file__))
        module_path = os.path.join(root, 'modules', '*.py')
        for file in glob.glob(module_path):
            modules.append(os.path.basename(file)[:-3])
        self.module_names = modules + I3S_SINGLE_NAMES + I3STATUS_MODULES
        self.container_modules = []

    def check_child_friendly(self, name):
        name = name.split()[0]
        if name in self.container_modules:
            return
        root = os.path.dirname(os.path.realpath(__file__))
        module_path = os.path.join(root, 'modules')
        info = imp.find_module(name, [module_path])
        if not info:
            return
        (file, pathname, description) = info
        try:
            py_mod = imp.load_module(name, file, pathname, description)
        except:
            # We cannot load the module!  We could error out here but then the
            # user gets informed that the problem is with their config.  This
            # is not correct.  Better to say that all is well and then the
            # config can get parsed and py3status loads.  The error about the
            # failing module load is better handled at that point, and will be.
            return
        if hasattr(py_mod, 'Py3status'):
            if hasattr(py_mod.Py3status, 'Meta'):
                if hasattr(py_mod.Py3status.Meta, 'container'):
                    self.container_modules.append(name)
                    del py_mod
                    return
        del py_mod
        self.error('Module `{}` cannot contain others'.format(name))

    def check_module_name(self, name, offset=0):
        if name in ['general']:
            return
        split_name = name.split()
        if split_name[0] not in self.module_names:
            self.current_token -= len(split_name) - offset
            self.error('Unknown module')
        if len(split_name) > 1 and split_name[0] in I3S_SINGLE_NAMES:
            self.current_token -= len(split_name) - 1 - offset
            self.error('Invalid name cannot have 2 tokens')
        if len(split_name) > 2:
            self.current_token -= len(split_name) - 2 - offset
            self.error('Invalid name cannot have more than 2 tokens')

    def error(self, msg, previous=False):
        '''
        Raise a ParseException.
        We provide information to help locate the error in the config to allow
        easy config debugging for users.  previous indicates that the error
        actually occurred at the end of the previous line.
        '''
        token = self.tokens[self.current_token - 1]
{'fraction_non_alphanumeric': 0.14341962791613347, 'fraction_numerical': 0.012501230436066543, 'mean_word_length': 2.3333333333333335, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2941486', 'n_tokens_mistral': 3286, 'n_tokens_neox': 3170, 'n_words': 818}