<commit_before>import os
import re

def load(*files):
    o = None
    for f in files:
        if os.path.isfile(f):
            o = Properties(f, o)
    return o

class Properties(dict):
    def __init__(self, path, parent=None):
        dict.__init__(self)

        if parent:
            self.update(parent)
            self.types = dict(parent.types)
        else:
            self.types = {}

        decoder = {
            'int': int,
            'bool': lambda a: a == 'true',
            'string': lambda a: a.replace('\:', ':').replace('\=', '='),
            'none': lambda a: None
        }

        c_seperator  = (':', '=')
        c_newline    = ('\n', '\r', '')
        c_whitespace = (' ', '\t', '\f')
        c_escapes    = ('t','n','r','f')
        c_comment    = ('#','!')

        f = open(path)
        while True:
            #skip whitespace
            c = f.read(1)
            while c in c_whitespace:
                c = f.read(1)

            #finish on EOF
            if c == '':
                break

            #skip comments
            if c in c_comment:
                f.readline()
                continue

            #skip blank lines
            if c in c_newline:
                continue

            #read key
            k = ""
            while True:
                if c in c_newline:
                    break

                elif c in c_seperator + c_whitespace:
                    c = f.read(1)
                    break

                elif c == '\\':
                    c = f.read(1)
                    if c in c_escapes:
                        k += ('\\'+c).decode('string-escape')
                    elif c == 'u':
                        k += unichr(int(f.read(4)))
                    else:
                        k += c

                else:
                    k += c

                c = f.read(1)


            #skip whitespace
            while c in c_whitespace:
                c = f.read(1)

            #read value
            v = ""
            while True:
                if c in c_newline:
                    break

                elif c == '\\':
                    c = f.read(1)
                    if c in c_escapes:
                        v += ('\\'+c).decode('string-escape')
                    elif c == 'u':
                        v += unichr(int(f.read(4)))
                    else:
                        v += c
                else:
                    v += c

                c = f.read(1)

            k = k.replace('-', '_')

            if re.match('^\-?\d+$', v):
                ty = 'int'
            elif v in ('true', 'false'):
                ty = 'bool'
            elif v != '':
                ty = 'string'
            elif k in self.types:
                ty = self.types[k]
            else:<commit_msg>properties: another substantial rewrite. this decoder is significantly cleaner/more error tolerant/less derpy<commit_after>import os
import re

def load(*files):
    o = None
    for f in files:
        if os.path.isfile(f):
            o = Properties(f, o)
    return o

class Properties(dict):
    def __init__(self, path, parent=None):
        dict.__init__(self)

        if parent:
            self.update(parent)
            self.types = dict(parent.types)
        else:
            self.types = {}

        decoder = {
            'int': int,
            'bool': lambda a: a == 'true',
            'string': lambda a: a.replace('\:', ':').replace('\=', '='),
            'none': lambda a: None
        }

        c_seperator  = (':', '=')
        c_whitespace = (' ', '\t', '\f')
        c_escapes    = ('t','n','r','f')
        c_comment    = ('#','!')

        r_unescaped  = '(?<!\\\\)(?:\\\\\\\\)*'
        r_whitespace = '[' + re.escape(''.join(c_whitespace)) + ']*'
        r_seperator  = r_whitespace + r_unescaped + '[' + re.escape(''.join(c_seperator + c_whitespace)) + ']' + r_whitespace

        f = open(path)
        d = f.read()
        f.close()

        #Deal with Windows / Mac OS linebreaks
        d = d.replace('\r\n','\n')
        d = d.replace('\r', '\n')
        #Strip leading whitespace
        d = re.sub('\n\s*', '\n', d, flags=re.MULTILINE)
        #Split logical lines
        d = re.split(r_unescaped+'\n', d, flags=re.MULTILINE)

        #This handles backslash escapes in keys/values
        def parse(input):
            token = list(input)
            out = ""

            while len(token) > 0:
                c = token.pop(0)
                if c == '\\':
                    try:
                        c = token.pop(0)
                        if c in c_escapes:
                            out += ('\\'+c).decode('string-escape')
                        elif c == 'u':
                            b = ""
                            for i in range(4):
                                b += token.pop(0)
                            out += unichr(int(b, 16))
                        else:
                            out += c
                    except IndexError:
                        raise ValueError("Invalid escape sequence in input: %s" % input)
                else:
                    out += c

            return out

        for line in d:
            #Strip comments and empty lines
            if line == '' or line[0] in c_comment:
                continue

            #Strip escaped newlines
            line = line.replace('\\\n', '')
            assert not '\n' in line

            #Split into k,v
            x = re.split(r_seperator, line, maxsplit=1)

            #No seperator, parse as empty value.
            if len(x) == 1:
                k, v = x[0], ""
            else:
                k, v = x

            k = parse(k).replace('-', '_')
            v = parse(v)

            if re.match('^\-?\d+$', v):
                ty = 'int'
            elif v in ('true', 'false'):
                ty = 'bool'
            elif v != '':
                ty = 'string'
            elif k in self.types:
                ty = self.types[k]
            else:
{'fraction_non_alphanumeric': 0.13639376864434868, 'fraction_numerical': 0.0034802784222737818, 'mean_word_length': 0.9709340300457218, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4365331', 'n_tokens_mistral': 1805, 'n_tokens_neox': 1696, 'n_words': 444}