<commit_before>import os
import re

def load(*files):
    o = None
    for f in files:
        if os.path.isfile(f):
            o = Properties(f, o)
    return o
        
class Properties(dict):
    def __init__(self, path, parent=None):
        dict.__init__(self)

        if parent:
            self.update(parent)
            self.types = dict(parent.types)
        else:
            self.types = {}
        
        decoder = {
            'int': int,
            'bool': lambda a: a == 'true',
            'string': lambda a: a.replace('\:', ':').replace('\=', '='),
            'none': lambda a: None
        }
        ty = None
        
        f = open(path)
        for l in f:
            #Comment
            if l.startswith('#'):
                continue
            
            #K,V pair
            m = re.match('^([^=]+)=(.*)$', l.strip())
            if m:
                k, v = m.groups()
                k = k.replace('-', '_')
                
                if re.match('^\-?\d+$', v):
                    ty = 'int'
                elif v in ('true', 'false'):
                    ty = 'bool'
                elif v != '':
                    ty = 'string'
                elif k in self.types:
                    ty = self.types[k]
                else:
                    ty = 'string'
                
                self.types[k] = ty
                self[k] = decoder[ty](v)

    def get_plugins(self):
        plugins = {}
        enabled = []
        for k, v in self.iteritems():
            m = re.match('^plugin\.(.+)\.(.+)$', k)
            if m:
                plugin, k2 = m.groups()
                
                if plugin not in plugins:
                    plugins[plugin] = {}
                
                if k2 == 'enabled':
                    if v:
                        enabled.append(plugin)
                else:
                    plugins[plugin][k2] = v

        for n in sorted(enabled):
            yield n, plugins[n]

    def get_jvm_options(self):<commit_msg>properties: more compliant implementation - supports multiline values, escape characters, unicode escapes<commit_after>import os
import re

def load(*files):
    o = None
    for f in files:
        if os.path.isfile(f):
            o = Properties(f, o)
    return o

class Properties(dict):
    def __init__(self, path, parent=None):
        dict.__init__(self)

        if parent:
            self.update(parent)
            self.types = dict(parent.types)
        else:
            self.types = {}

        decoder = {
            'int': int,
            'bool': lambda a: a == 'true',
            'string': lambda a: a.replace('\:', ':').replace('\=', '='),
            'none': lambda a: None
        }

        c_seperator  = (':', '=')
        c_newline    = ('\n', '\r', '')
        c_whitespace = (' ', '\t', '\f')
        c_escapes    = ('t','n','r','f')
        c_comment    = ('#','!')

        f = open(path)
        while True:
            #skip whitespace
            c = f.read(1)
            while c in c_whitespace:
                c = f.read(1)

            #finish on EOF
            if c == '':
                break

            #skip comments
            if c in c_comment:
                f.readline()
                continue

            #skip blank lines
            if c in c_newline:
                continue

            #read key
            k = ""
            while True:
                if c in c_newline:
                    break

                elif c in c_seperator + c_whitespace:
                    c = f.read(1)
                    break

                elif c == '\\':
                    c = f.read(1)
                    if c in c_escapes:
                        k += ('\\'+c).decode('string-escape')
                    elif c == 'u':
                        k += unichr(int(f.read(4)))
                    else:
                        k += c

                else:
                    k += c

                c = f.read(1)


            #skip whitespace
            while c in c_whitespace:
                c = f.read(1)

            #read value
            v = ""
            while True:
                if c in c_newline:
                    break

                elif c == '\\':
                    c = f.read(1)
                    if c in c_escapes:
                        v += ('\\'+c).decode('string-escape')
                    elif c == 'u':
                        v += unichr(int(f.read(4)))
                    else:
                        v += c
                else:
                    v += c

                c = f.read(1)

            k = k.replace('-', '_')

            if re.match('^\-?\d+$', v):
                ty = 'int'
            elif v in ('true', 'false'):
                ty = 'bool'
            elif v != '':
                ty = 'string'
            elif k in self.types:
                ty = self.types[k]
            else:
                ty = 'string'

            self.types[k] = ty
            self[k] = decoder[ty](v)
        f.close()

    def get_plugins(self):
        plugins = {}
        enabled = []
        for k, v in self.iteritems():
            m = re.match('^plugin\.(.+)\.(.+)$', k)
            if m:
                plugin, k2 = m.groups()
                
                if plugin not in plugins:
                    plugins[plugin] = {}
                
                if k2 == 'enabled':
                    if v:
                        enabled.append(plugin)
                else:
                    plugins[plugin][k2] = v

        for n in sorted(enabled):
            yield n, plugins[n]

    def get_jvm_options(self):
{'fraction_non_alphanumeric': 0.12769339456022608, 'fraction_numerical': 0.00282585658777817, 'mean_word_length': 0.8826462765957447, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2793459', 'n_tokens_mistral': 1634, 'n_tokens_neox': 1520, 'n_words': 404}