<commit_before>package parse

import java.io.File


fun parseJavaFile(file: File): AnalyzedJavaFile {
    val source = file.readText()

    val importMatches = "(?:import.*;)"
            .toRegex(RegexOption.MULTILINE)
            .findAll(source)

    val builder = StringBuilder();
    for (match in importMatches.iterator()) {
        val groups = match.groups
        if (groups.size > 0) {
            builder.append(groups[0]?.value?.trim())
            builder.append("\n")
        }
    }
    val imports = builder.toString()


    // findAll runs out of memory
    val methodMatches = mutableListOf<MatchResult>()
    var pos = 0
    while (true) {
        // for regex101 https://regex101.com/r/D7gTv3/1
        // (?:\n*?((?:\h*?\/\*\*\h*[\w@]*\n*){1}(?:\h*\*.*\n*)*?(?:\h*\*\/){1})\n)*((?:^\h*\@.*\n)*?)?\h*((?:\w* )*)(.* )?([\w\.]*(?:\[\])?) (\w*)\(((?:\n*[^\)]*)*)\)([^\{]*)\{((?:.*\n)(?:\n*(?:        .*\n))*)\    \}\n*
        val methodsMatch = "(?:\n*?((?:\\h*?\\/\\*\\*\\h*[\\w@]*\n*){1}(?:\\h*\\*.*\n*)*?(?:\\h*\\*\\/){1})\n)*((?:^\\h*\\@.*\n)*?)?\\h*((?:\\w* )*)(.* )?([\\w\\.]*(?:\\[\\])?) (\\w*)\\(((?:\n*[^\\)]*)*)\\)([^\\{]*)\\{((?:.*\n)(?:\n*(?:        .*\n))*)\\    \\}\n*"
                .toRegex(RegexOption.MULTILINE)
                .find(source, pos)

        if (methodsMatch == null) {
            break
        }
        methodMatches.add(methodsMatch)
        pos = methodsMatch.range.last
    }<commit_msg>Match return type LiveData<LifecycleOwner><commit_after>package parse

import java.io.File


fun parseJavaFile(file: File): AnalyzedJavaFile {
    val source = file.readText()

    val importMatches = "(?:import.*;)"
            .toRegex(RegexOption.MULTILINE)
            .findAll(source)

    val builder = StringBuilder();
    for (match in importMatches.iterator()) {
        val groups = match.groups
        if (groups.size > 0) {
            builder.append(groups[0]?.value?.trim())
            builder.append("\n")
        }
    }
    val imports = builder.toString()


    // findAll runs out of memory
    val methodMatches = mutableListOf<MatchResult>()
    var pos = 0
    while (true) {
        // for regex101 https://regex101.com/r/D7gTv3/2
        // (?:\n*?((?:\h*?\/\*\*\h*[\w@]*\n*){1}(?:\h*\*.*\n*)*?(?:\h*\*\/){1})\n)*((?:^\h*\@.*\n)*?)?\h*((?:\w* )*)(.* )?([\w\.<>]*(?:\[\])?) (\w*)\(((?:\n*[^\)]*)*)\)([^\{]*)\{((?:.*\n)(?:\n*(?:        .*\n))*)\    \}\n*
        val methodsMatch = "(?:\n*?((?:\\h*?\\/\\*\\*\\h*[\\w@]*\n*){1}(?:\\h*\\*.*\n*)*?(?:\\h*\\*\\/){1})\n)*((?:^\\h*\\@.*\n)*?)?\\h*((?:\\w* )*)(.* )?([\\w\\.<>]*(?:\\[\\])?) (\\w*)\\(((?:\n*[^\\)]*)*)\\)([^\\{]*)\\{((?:.*\n)(?:\n*(?:        .*\n))*)\\    \\}\n*"
                .toRegex(RegexOption.MULTILINE)
                .find(source, pos)

        if (methodsMatch == null) {
            break
        }
        methodMatches.add(methodsMatch)
        pos = methodsMatch.range.last
    }
{'fraction_non_alphanumeric': 0.3359562093739309, 'fraction_numerical': 0.01094765651727677, 'mean_word_length': 1.9655172413793103, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3307464', 'n_tokens_mistral': 1169, 'n_tokens_neox': 1111, 'n_words': 147}