<commit_before>import datetime

def loads(s):
    """Returns a dictionary containing s, a string, parsed as toml."""
    retval = {}
    currentlevel = retval
    if isinstance(s, str):
        sl = list(s)
        openarr = 0
        openstring = False
        beginline = True
        keygroup = False
        delnum = 1
        for i in xrange(len(sl)):
            if sl[i] == '"' and sl[i-1] != '\\':
                openstring = not openstring
            if sl[i] == '#' and not openstring and not keygroup:
                j = i
                while sl[j] != '\n':
                    sl.insert(j, ' ')
                    sl.pop(j+1)
                    j += 1
            if sl[i] == '[' and not openstring:
                if beginline:
                    keygroup = True
                else:
                    openarr += 1
            if sl[i] == ']' and not openstring:
                if keygroup:
                    keygroup = False
                else:
                    openarr -= 1
            if sl[i] == '\n':
                if openarr:
                    sl.insert(i, ' ')
                    sl.pop(i+1)
                else:
                    beginline = True
            elif beginline and sl[i] != ' ' and sl[i] != '\t':
                beginline = False
        s = ''.join(sl)
        s = s.split('\n')
    else:
        raise Exception("What exactly are you trying to pull?")
    for line in s:
        line = line.strip()
        if line == "":
            continue
        if line[0] == '[':
            groups = line[1:-1].split('.')
            currentlevel = retval
            for i in xrange(len(groups)):
                group = groups[i]
                try:
                    currentlevel[group]
                    if i == len(groups) - 1:
                        raise Exception("What? "+group+" already exists?"+str(currentlevel))
                except KeyError:
                    currentlevel[group] = {}
                currentlevel = currentlevel[group]
        elif "=" in line:
            pair = line.split('=', 1)
            for i in xrange(len(pair)):
                pair[i] = pair[i].strip()
            value = load_value(pair[1])
            currentlevel[pair[0]] = value
    return retval

def load_value(v):
    if v == 'true':
        return True
    elif v == 'false':
        return False
    elif v[0] == '"':
        escapes = ['0', 'n', 'r', 't', '"', '\\']
        escapedchars = ['\0', '\n', '\r', '\t', '\"', '\\']
        escapeseqs = v.split('\\')[1:]
        for i in escapeseqs:
            if i[0] not in escapes and i[0] != 'x':
                raise Exception("Reserved escape sequence used")
        if "\\x" in v:
            hexchars = ['0', '1', '2', '3', '4', '5', '6', '7',<commit_msg>Make sure parser fails on errors in the hard toml file in mojombo/toml<commit_after>import datetime

def loads(s):
    """Returns a dictionary containing s, a string, parsed as toml."""
    retval = {}
    currentlevel = retval
    if isinstance(s, str):
        sl = list(s)
        openarr = 0
        openstring = False
        beginline = True
        keygroup = False
        delnum = 1
        for i in xrange(len(sl)):
            if sl[i] == '"' and sl[i-1] != '\\':
                openstring = not openstring
            if sl[i] == '#' and not openstring and not keygroup:
                j = i
                while sl[j] != '\n':
                    sl.insert(j, ' ')
                    sl.pop(j+1)
                    j += 1
            if sl[i] == '[' and not openstring:
                if beginline:
                    keygroup = True
                else:
                    openarr += 1
            if sl[i] == ']' and not openstring:
                if keygroup:
                    keygroup = False
                else:
                    openarr -= 1
            if sl[i] == '\n':
                if openarr:
                    if openstring:
                        raise Exception("Unbalanced quotes")
                    sl.insert(i, ' ')
                    sl.pop(i+1)
                else:
                    beginline = True
            elif beginline and sl[i] != ' ' and sl[i] != '\t':
                beginline = False
        s = ''.join(sl)
        s = s.split('\n')
    else:
        raise Exception("What exactly are you trying to pull?")
    for line in s:
        line = line.strip()
        if line == "":
            continue
        if line[0] == '[':
            line = line[1:].split(']', 1)
            if line[1].strip() != "":
                raise Exception("Key group not on a line by itself.")
            line = line[0]
            groups = line.split('.')
            currentlevel = retval
            for i in xrange(len(groups)):
                group = groups[i]
                try:
                    currentlevel[group]
                    if i == len(groups) - 1:
                        raise Exception("What? "+group+" already exists?"+str(currentlevel))
                except KeyError:
                    currentlevel[group] = {}
                currentlevel = currentlevel[group]
        elif "=" in line:
            pair = line.split('=', 1)
            for i in xrange(len(pair)):
                pair[i] = pair[i].strip()
            value = load_value(pair[1])
            currentlevel[pair[0]] = value
    return retval

def load_value(v):
    if v == 'true':
        return True
    elif v == 'false':
        return False
    elif v[0] == '"':
        testv = v[1:].split('"')
        closed = False
        for tv in testv:
            if tv == '':
                closed = True
            elif tv[-1] != '\\':
                if closed:
                    raise Exception("Stuff after closed string. WTF?")
                else:
                    closed = True
        escapes = ['0', 'n', 'r', 't', '"', '\\']
        escapedchars = ['\0', '\n', '\r', '\t', '\"', '\\']
        escapeseqs = v.split('\\')[1:]
        for i in escapeseqs:
            if i[0] not in escapes and i[0] != 'x':
                raise Exception("Reserved escape sequence used")
        if "\\x" in v:
            hexchars = ['0', '1', '2', '3', '4', '5', '6', '7',
{'fraction_non_alphanumeric': 0.13850861652440005, 'fraction_numerical': 0.009985504912224191, 'mean_word_length': 1.1705697308633345, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4052238', 'n_tokens_mistral': 1805, 'n_tokens_neox': 1722, 'n_words': 511}