<commit_before>import re
from subprocess import Popen, PIPE


def formatToBashStr(outputStr):
    return formatStr(outputStr)


def formatToVimStr(outputStr):
    return formatStr(outputStr)


def getDefinition(argsList):
    argsList.insert(0, "-n")
    argsList.insert(0, "sdcv")
    (definition, error) = Popen(argsList, stdout=PIPE).communicate()
    return definition


def formatStr(outputStr):
    replacedBullet = 1
    replacedStr = ""
    wordMeaningPattern = "^\\-\\s+.*"
    wordExamplePattern = "^\\=.*"
    wordMultiExamplesPattern = "^\\!.*"
    wordPattern = "^\\@.*"
    dictNamePattern = "^\\-\\-\\>.*"
    finalStr = ""

    startLineCharIdx = -1
    prevLineCharIdx = -1
    while True:
        endLineCharIdx = outputStr.find('\n', startLineCharIdx + 1)

        if (endLineCharIdx < 0):
            break

        # Also include newline as part of the extracted string
        line = outputStr[startLineCharIdx + 1:endLineCharIdx + 1]

        # The order of the if/elseif statements matter to the logic flow of
        # this function
        if (re.match(wordExamplePattern, line)):
            # Re-format WordExample
            replacedStr = re.sub("^\\=\\s*", "\\t- ", line,
                    flags=re.IGNORECASE)
            replacedStr = re.sub("^\\+\\s*", ": ", replacedStr,
                    flags=re.IGNORECASE)
            finalStr += replacedStr

        elif (re.match(wordMeaningPattern, line)):
            # Re-format WordMeaning
            if (prevLineCharIdx > -1):
                prevLine = outputStr[prevLineCharIdx + 1:startLineCharIdx + 1]

                if (re.match(wordMultiExamplesPattern, prevLine)):
                    replacedStr = re.sub("^\\-", "\\t\\t-", line,
                            flags=re.IGNORECASE)
                else:
                    replacedStr = re.sub("^\\-\\s+", replacedBullet + ". ",
                            line, flags=re.IGNORECASE)
                    ++replacedBullet
            finalStr += replacedStr

        elif (re.match(wordPattern, line)):
            # Re-format Word
            replacedStr = re.sub("^\\@", "", line, flags=re.IGNORECASE)
            finalStr += replacedStr
            replacedBullet = 1

        prevLineCharIdx = startLineCharIdx
        startLineCharIdx = endLineCharIdx

    replacedBullet = 1
    replacedStr = ""
    startLineCharIdx = -1
    finalStr2 = ""

    while True:
        endLineCharIdx = finalStr.find('\n', startLineCharIdx + 1)

        if (endLineCharIdx < 0):
            break

        # Also include newline as part of the extracted string
        line = finalStr[startLineCharIdx + 1:endLineCharIdx + 1]

        if (re.match(wordMultiExamplesPattern, line)):
            replacedStr = re.sub("^\\!(.*)", "\\t- \\1:", line,
                    flags=re.IGNORECASE)
            finalStr2 += replacedStr

        elif (re.match(dictNamePattern, line)):
            replacedStr = re.sub("^\\-\\-\\>", "@Dictionary: ", line,
                    re.IGNORECASE)
            finalStr2 += replacedStr

            startLineCharIdx = endLineCharIdx
            endLineCharIdx = finalStr.find('\n', startLineCharIdx + 1)
            line = finalStr[startLineCharIdx + 1:endLineCharIdx + 1]

            replacedStr = re.sub("^\\-\\-\\>", "@SearchedTerm: ", line,
                    re.IGNORECASE)
            finalStr2 += replacedStr<commit_msg>- Implementing getDefinition<commit_after>import re
from subprocess import Popen, PIPE


def getDefinition(argsList, caller="vim"):
    argsList.insert(0, "-n")
    argsList.insert(0, "sdcv")
    (definition, error) = Popen(argsList, stdout=PIPE).communicate()
    definition = formatStr(definition)

    if (caller == "bash"):
        stardictResult = "[A-Z].*"
        stardictWord = "[a-z][^/]*"
        stardictWordType = "\\*.*"
        stardictWordMeaning = "[0-9].*"
        stardictWordExample = "(\\t\\-\\s.*\\:|\\!.*)"
        stardictDictName = "\\@.*"
        # The color codes were generated by running the following commands
        # in bash:
        #
        # for (( i = 0; i < 17; ++i ));
        # do echo "$(tput setaf $i)This is ($i) $(tput sgr0)";
        # done
        #
        preProcColorCode = "$(tput setaf 9)\\1$(tput sgr0)"        # 9
        errorColorCode = "$(tput setaf 9)\\1$(tput srg0)"          # 9
        statementColorCode = "$(tput setaf 2)\\1$(tput srg0)"      # 2
        identifierColorCode = "$(tput setaf 4)\\1$(tput srg0)"     # 4
        typeColorCode = "$(tput setaf 3)\\1$(tput srg0)"           # 3
        underlinedColorCode = "$(tput setaf 13)\\1$(tput srg0)"    # 13

        finalStr = ""
        replacedStr = ""
        startLineCharIdx = -1

        while True:
            endLineCharIdx = definition.find('\n', startLineCharIdx + 1)

            if (endLineCharIdx < 0):
                break

            # Also include newline as part of the extracted string
            line = definition[startLineCharIdx + 1:endLineCharIdx + 1]

            if (re.match("^" + stardictResult, line)):
                # Re-format stardictResult
                replacedStr = re.sub("^(" + stardictResult + ")",
                        preProcColorCode, line, flags=re.IGNORECASE)

            elif (re.match("^" + stardictWord, line)):
                # Re-format stardictWord
                replacedStr = re.sub("^(" + stardictWord + ")", errorColorCode,
                        line, flags=re.IGNORECASE)

            elif (re.match("^" + stardictWordType, line)):
                # Re-format stardictWordType
                replacedStr = re.sub("^(" + stardictWordType + ")",
                        statementColorCode, line, flags=re.IGNORECASE)

            elif (re.match("^" + stardictWordMeaning, line)):
                # Re-format stardictWordMeaning
                replacedStr = re.sub("^(" + stardictWordMeaning + ")",
                        identifierColorCode, line, flags=re.IGNORECASE)

            elif (re.match("^" + stardictWordExample, line)):
                # Re-format stardictWordExample
                replacedStr = re.sub("^(" + stardictWordExample + ")",
                        typeColorCode, line, flags=re.IGNORECASE)

            elif (re.match("^" + stardictDictName, line)):
                # Re-format stardictDictName
                replacedStr = re.sub("^(" + stardictDictName + ")",
                        underlinedColorCode, line, flags=re.IGNORECASE)
            else:
                replacedStr = line

            finalStr += replacedStr

        return finalStr

    return definition


def formatStr(outputStr):
    replacedBullet = 1
    wordMeaningPattern = "^\\-\\s+.*"
    wordExamplePattern = "^\\=.*"
    wordMultiExamplesPattern = "^\\!.*"
    wordPattern = "^\\@.*"
    dictNamePattern = "^\\-\\-\\>.*"
    finalStr = ""

    startLineCharIdx = -1
    prevLineCharIdx = -1

    while True:
        endLineCharIdx = outputStr.find('\n', startLineCharIdx + 1)

        if (endLineCharIdx < 0):
            break

        # Also include newline as part of the extracted string
        line = outputStr[startLineCharIdx + 1:endLineCharIdx + 1]

        # The order of the if/elseif statements matter to the logic flow of
        # this function
        if (re.match(wordExamplePattern, line)):
            # Re-format WordExample
            replacedStr = re.sub("^\\=\\s*", "\\t- ", line,
                    flags=re.IGNORECASE)
            replacedStr = re.sub("^\\+\\s*", ": ", replacedStr,
                    flags=re.IGNORECASE)
            finalStr += replacedStr

        elif (re.match(wordMeaningPattern, line)):
            # Re-format WordMeaning
            if (prevLineCharIdx > -1):
                prevLine = outputStr[prevLineCharIdx + 1:startLineCharIdx + 1]

                if (re.match(wordMultiExamplesPattern, prevLine)):
                    replacedStr = re.sub("^\\-", "\\t\\t-", line,
                            flags=re.IGNORECASE)
                else:
                    replacedStr = re.sub("^\\-\\s+", replacedBullet + ". ",
                            line, flags=re.IGNORECASE)
                    ++replacedBullet
            finalStr += replacedStr

        elif (re.match(wordPattern, line)):
            # Re-format Word
            replacedStr = re.sub("^\\@", "", line, flags=re.IGNORECASE)
            finalStr += replacedStr
            replacedBullet = 1
        else:
            finalStr += line

        prevLineCharIdx = startLineCharIdx
        startLineCharIdx = endLineCharIdx

    finalStr2 = ""
    replacedBullet = 1
    startLineCharIdx = -1

    while True:
        endLineCharIdx = finalStr.find('\n', startLineCharIdx + 1)

        if (endLineCharIdx < 0):
            break

        # Also include newline as part of the extracted string
        line = finalStr[startLineCharIdx + 1:endLineCharIdx + 1]

        if (re.match(wordMultiExamplesPattern, line)):
            replacedStr = re.sub("^\\!(.*)", "\\t- \\1:", line,
                    flags=re.IGNORECASE)
            finalStr2 += replacedStr

        elif (re.match(dictNamePattern, line)):
            replacedStr = re.sub("^\\-\\-\\>", "@Dictionary: ", line,
                    re.IGNORECASE)
            finalStr2 += replacedStr

            startLineCharIdx = endLineCharIdx
            endLineCharIdx = finalStr.find('\n', startLineCharIdx + 1)
            line = finalStr[startLineCharIdx + 1:endLineCharIdx + 1]

            replacedStr = re.sub("^\\-\\-\\>", "@SearchedTerm: ", line,
                    re.IGNORECASE)
            finalStr2 += replacedStr
        else:
            finalStr2 += line
{'fraction_non_alphanumeric': 0.13162819713038054, 'fraction_numerical': 0.009565398211686421, 'mean_word_length': 1.7720461095100866, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5479770', 'n_tokens_mistral': 2949, 'n_tokens_neox': 2803, 'n_words': 601}