<commit_before>function nextToken(source, offset, stack) {
    var skip = 0
    while (/\s/.test(source.charAt(offset + skip))) {
        skip += 1
    }

    var char = source.charAt(offset + skip)
    if (["(", ")", ",", ";"].indexOf(char) >= 0) {
        return { token: char, skip: skip + 1 }
    }

    var token = ""
    while (/\S/.test(source.charAt(offset + skip))) {
        var char = source.charAt(offset + skip)
        if (["(", ")", ",", ";"].indexOf(char) >= 0) break
        token += char
        skip += 1
    }

    return {
        token: token,
        skip: skip,
    }
}

function parseClassMember(source, offset, stack) {
    var line = "    "
    var returnType = ""
    var t = null
    while (t = nextToken(source, offset, stack)) {
        offset += t.skip
        if (["abstract", "public", "protected", "static"].indexOf(t.token) >= 0) {
            line += t.token + " "
        } else if (t.token !== "final") {
            returnType = t.token
            break
        }
    }

    if (returnType === "{}") {
        return {
            source: source,
            offset: offset + 1,
            stack: stack,
        }
    }

    t = nextToken(source, offset, stack)
    line += t.token
    offset += t.skip

    t = nextToken(source, offset, stack)
    if (t.token === "(") {
        offset += t.skip
        line += "("
        var i = 0
        while (t = nextToken(source, offset, stack)) {
            offset += t.skip
            if (t.token === ")") {
                line += ")"
                break
            } else if (t.token === ",") {
                line += ", "
            } else {
                line += "arg" + i + ": " + t.token
                i += 1
            }
        }
        t = nextToken(source, offset, stack)
    }
    if (t.token === ";") {
        offset += t.skip
    } else {
        return null
    }

    line += ": " + returnType
    stack.push(line)
    return {
        source: source,
        offset: offset,
        stack: stack,
    }
}

function parseClass(source, offset, stack, classType) {
    var t = nextToken(source, offset, stack)
    if (t.token !== "class") return null
    offset += t.skip

    t = nextToken(source, offset, stack)
    var className = t.token
    offset += t.skip

    var line = classType + "class " + className.split(".").reverse()[0]
    while (t = nextToken(source, offset, stack)) {
        line += " " + t.token
        offset += t.skip
        if (t.token === "{") break
    }
    stack.push(line)

    while (t = nextToken(source, offset, stack)) {
        if (t.token === "}") {
            offset += t.skip
            stack.push("}")
            break
        } else if (t.token === className) {
            offset += t.skip
            var line = "    constructor"
            var i = 0
            while (t = nextToken(source, offset, stack)) {
                offset += t.skip
                if (t.token === "(") {
                    line += "("
                } else if (t.token === ")") {
                    line += ")"
                    break
                } else if (t.token === ",") {
                    line += ", "
                } else {
                    line += "arg" + i + ": " + t.token
                    i += 1
                }
            }
            stack.push(line)
            offset += 1     // ;
        } else {
            var context = parseClassMember(source, offset, stack)
            if (!context) return null
            source = context.source
            offset = context.offset
            stack = context.stack
        }
    }

    return {
        source: source,
        offset: offset,
        stack: stack,
    }
}

function parseInterface(source, offset, stack, type) {

}

function parse(source, offset, stack) {
    var type = ""
    while (offset < source.length) {
        var next = nextToken(source, offset, stack)
        var token = next.token
        var skip = next.skip
        if (token === "final") {
            offset += skip
        } else if (token === "abstract" || token === "public") {
            type += token + " "
            offset += skip
        } else if (token === "class") {
            context = parseClass(source, offset, stack, type)
            if (!context) return { stack: stack }
            source = context.source
            offset = context.offset
            stack = context.stack
            type = ""
        } else if (token === "interface") {
            context = parseInterface(source, offset, stack, type)
            if (!context) return { stack: stack }
            source = context.source
            offset = context.offset
            stack = context.stack
            type = ""
        } else {
            offset += skip
        }
    }
    return { stack: stack }
}

module.exports = function(source) {
    return parse(source || test, 0, []).stack.join("\n")
}

var test = `
            public final class org.pmw.tinylog.Configuration {
              org.pmw.tinylog.Configuration(org.pmw.tinylog.Configurator, org.pmw.tinylog.Level, java.util.Map<java.lang.String, org.pmw.tinylog.Level>, java.lang.String, java.util.Locale, java.util.List<org.pmw.tinylog.WriterDefinition>, org.pmw.tinylog.WritingThread, java.lang.Integer);
              public org.pmw.tinylog.Level getLevel();
              public boolean hasCustomLevels();
              public org.pmw.tinylog.Level getLevel(java.lang.String);
              public java.lang.String getFormatPattern();
              public java.util.Locale getLocale();
              public java.util.List<org.pmw.tinylog.writers.Writer> getWriters();
              public org.pmw.tinylog.WritingThread getWritingThread();
              public int getMaxStackTraceElements();
              org.pmw.tinylog.Configurator getConfigurator();
              boolean isOutputPossible(org.pmw.tinylog.Level);
              org.pmw.tinylog.writers.Writer[] getEffectiveWriters(org.pmw.tinylog.Level);
              java.util.List<org.pmw.tinylog.Token>[] getEffectiveFormatTokens(org.pmw.tinylog.Level);
              java.util.Set<org.pmw.tinylog.writers.LogEntryValue> getRequiredLogEntryValues(org.pmw.tinylog.Level);
              org.pmw.tinylog.StackTraceInformation getRequiredStackTraceInformation(org.pmw.tinylog.Level);
              static {};
            }
`<commit_msg>Add typeMap<commit_after>var typeMap = {
    "byte": "number",
    "short": "number",
    "int": "number",
    "long": "number",
    "float": "number",
    "double": "number",
    "char": "any",
}

function nextToken(source, offset, stack) {
    var skip = 0
    while (/\s/.test(source.charAt(offset + skip))) {
        skip += 1
    }

    var char = source.charAt(offset + skip)
    if (["(", ")", ",", ";"].indexOf(char) >= 0) {
        return { token: char, skip: skip + 1 }
    }

    var token = ""
    while (/\S/.test(source.charAt(offset + skip))) {
        var char = source.charAt(offset + skip)
        if (["(", ")", ",", ";"].indexOf(char) >= 0) break
        token += char
        skip += 1
    }

    return {
        token: token,
        skip: skip,
    }
}

function parseClassMember(source, offset, stack) {
    var line = "    "
    var returnType = ""
    var t = null
    while (t = nextToken(source, offset, stack)) {
        offset += t.skip
        if (["abstract", "public", "protected", "static"].indexOf(t.token) >= 0) {
            line += t.token + " "
        } else if (t.token !== "final") {
            returnType = t.token
            break
        }
    }

    if (returnType === "{}") {
        return {
            source: source,
            offset: offset + 1,
            stack: stack,
        }
    }

    t = nextToken(source, offset, stack)
    line += t.token
    offset += t.skip

    t = nextToken(source, offset, stack)
    if (t.token === "(") {
        offset += t.skip
        line += "("
        var i = 0
        while (t = nextToken(source, offset, stack)) {
            offset += t.skip
            if (t.token === ")") {
                line += ")"
                break
            } else if (t.token === ",") {
                line += ", "
            } else {
                var type = typeMap[t.token] || t.token
                line += "arg" + i + ": " + type
                i += 1
            }
        }
        t = nextToken(source, offset, stack)
    }
    if (t.token === ";") {
        offset += t.skip
    } else {
        return null
    }

    line += ": " + (typeMap[returnType] || returnType)
    stack.push(line)
    return {
        source: source,
        offset: offset,
        stack: stack,
    }
}

function parseClass(source, offset, stack, classType) {
    var t = nextToken(source, offset, stack)
    if (t.token !== "class") return null
    offset += t.skip

    t = nextToken(source, offset, stack)
    var className = t.token
    offset += t.skip

    var line = classType + "class " + className.split(".").reverse()[0]
    while (t = nextToken(source, offset, stack)) {
        line += " " + t.token
        offset += t.skip
        if (t.token === "{") break
    }
    stack.push(line)

    while (t = nextToken(source, offset, stack)) {
        if (t.token === "}") {
            offset += t.skip
            stack.push("}")
            break
        } else if (t.token === "public") {
            offset += t.skip
        } else if (t.token === className) {
            offset += t.skip
            var line = "    constructor"
            var i = 0
            while (t = nextToken(source, offset, stack)) {
                offset += t.skip
                if (t.token === "(") {
                    line += "("
                } else if (t.token === ")") {
                    line += ")"
                    break
                } else if (t.token === ",") {
                    line += ", "
                } else {
                    var type = typeMap[t.token] || t.token
                    line += "arg" + i + ": " + type
                    i += 1
                }
            }
            stack.push(line)
            offset += 1     // ;
        } else {
            var context = parseClassMember(source, offset, stack)
            if (!context) return null
            source = context.source
            offset = context.offset
            stack = context.stack
        }
    }

    return {
        source: source,
        offset: offset,
        stack: stack,
    }
}

function parseInterface(source, offset, stack, type) {

}

function parse(source, offset, stack) {
    var type = ""
    while (offset < source.length) {
        var next = nextToken(source, offset, stack)
        var token = next.token
        var skip = next.skip
        if (token === "final") {
            offset += skip
        } else if (token === "abstract" || token === "public") {
            type += token + " "
            offset += skip
        } else if (token === "class") {
            context = parseClass(source, offset, stack, type)
            if (!context) return { stack: stack }
            source = context.source
            offset = context.offset
            stack = context.stack
            type = ""
        } else if (token === "interface") {
            context = parseInterface(source, offset, stack, type)
            if (!context) return { stack: stack }
            source = context.source
            offset = context.offset
            stack = context.stack
            type = ""
        } else {
            offset += skip
        }
    }
    return { stack: stack }
}

module.exports = function(source) {
    return parse(source, 0, []).stack.join("\n")
}
{'fraction_non_alphanumeric': 0.14456484348125215, 'fraction_numerical': 0.0025799793601651187, 'mean_word_length': 1.492818863879957, 'pattern_counts': {'":': 13, '<': 10, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7345265', 'n_tokens_mistral': 3383, 'n_tokens_neox': 3316, 'n_words': 817}