<commit_before>import os
import re
import sys

from textwrap import TextWrapper
from difflib import Differ

desc_column = 34
cr = re.compile("(?P<comment_opening>\\/\\*+\n)"
                "(?P<comment_body>(.|\n(?!\\*\\/))*?)"
                "\n *\\*+\\/\n"
                "(?P<next_line>.*?\\()")

# matches (and will remove) the rfoptional doxygen tag that is no longer used
optional_re = re.compile("\\\\rfoptional\\{.*?\\}")

param_re = re.compile("( *\\*+ )(?P<param_name>(@param\\[?.*?\\]? (\w|\\.)+?)|(@return))\s(?P<param_desc>.*?)(?=( *\\*+ @param)|( *\\*+ @return)|( *\\*+ @see)|( *\\*+\\/))", flags=re.DOTALL)

# should match newline plus the start of next line of comment
stars_re = re.compile("(((\n)|(\r\n)|(\n\r)|(\r))|^)( *\\*+)(?!\\/)")

cm_intro = "/**"
cm_outro = "\n**/\n"

wrapper = TextWrapper(width=80,
                      initial_indent='',
                      subsequent_indent=' **' + ' ' * (desc_column - 3),
                      break_long_words=True,
                      break_on_hyphens=True)

diff = Differ()


def process_generic_comment_lines(s):
    s = stars_re.sub("\n **", s)
    return s

def process_parameter(param_name, param_desc):
    ret = " ** "
    ret += param_name
    length = len(ret)

    # Remove all newline and comment stars
    param_desc = stars_re.sub("", param_desc)
    # Remove any extra spaces
    param_desc = " ".join(param_desc.split())

    #perform the line wrapping
    wrapper.initial_indent = " " * (desc_column - length)
    if desc_column - length <= 0:
        wrapper.initial_indent = "\n" + wrapper.subsequent_indent
    ret += wrapper.fill(param_desc)
    return ret + "\n"


def process_comment(comment):
    comment = optional_re.sub("", comment)
    m = param_re.search(comment)

    if m is None:  #no parameters found for formatting
        return "{}{}{}".format(
            cm_intro,
            process_generic_comment_lines(comment),
            cm_outro
        )

    ret = cm_intro
    while m is not None:
        ret += process_generic_comment_lines(comment[:m.start()])
        ret += process_parameter(m.group("param_name"), m.group("param_desc"))
        comment = comment[m.end():]
        m = param_re.search(comment)

    # the added newline after the last parameter should go away
    # TODO: find a better way to do this
    ret = ret[:-1]

    ret += cm_outro
    return ret
<commit_msg>Fixing regex for comment styling<commit_after>import os
import re
import sys

from textwrap import TextWrapper
from difflib import Differ

desc_column = 34
cr = re.compile("(?P<comment_opening>\\/\\*+\n)"
                "(?P<comment_body>(.|\n(?!\\*\\/))*?)"
                "\n *\\*+\\/\n"
                "(?P<next_line>.*?\\()")

# matches (and will remove) the rfoptional doxygen tag that is no longer used
optional_re = re.compile("\\\\rfoptional\\{.*?\\}")

param_re = re.compile("( *\\*+ )(?P<param_name>(@param\\[?.*?\\]? (\w|\\.)+?)|(@return))\s(?P<param_desc>.*?)(?=( *\\*+ @param)|( *\\*+ @return)|( *\\*+ @see)|( *\\*+\\/)|$)", flags=re.DOTALL)

# should match newline plus the start of next line of comment
stars_re = re.compile("(((\n)|(\r\n)|(\n\r)|(\r))|^)( *\\*+)(?!\\/)")

cm_intro = "/**"
cm_outro = "\n**/\n"

wrapper = TextWrapper(width=80,
                      initial_indent='',
                      subsequent_indent=' **' + ' ' * (desc_column - 3),
                      break_long_words=True,
                      break_on_hyphens=True)

diff = Differ()


def process_generic_comment_lines(s):
    s = stars_re.sub("\n **", s)
    return s

def process_parameter(param_name, param_desc):
    ret = " ** "
    ret += param_name
    length = len(ret)

    # Remove all newline and comment stars
    param_desc = stars_re.sub("", param_desc)
    # Remove any extra spaces
    param_desc = " ".join(param_desc.split())

    #perform the line wrapping
    wrapper.initial_indent = " " * (desc_column - length)
    if desc_column - length <= 0:
        wrapper.initial_indent = "\n" + wrapper.subsequent_indent
    ret += wrapper.fill(param_desc)
    return ret + "\n"


def process_comment(comment):
    comment = optional_re.sub("", comment)
    m = param_re.search(comment)

    if m is None:  #no parameters found for formatting
        return "{}{}{}".format(
            cm_intro,
            process_generic_comment_lines(comment),
            cm_outro
        )

    ret = cm_intro
    while m is not None:
        ret += process_generic_comment_lines(comment[:m.start()])
        ret += process_parameter(m.group("param_name"), m.group("param_desc"))
        comment = comment[m.end():]
        m = param_re.search(comment)

    # add the remaining comment if any or remove the last newline added if not
    if comment:
        ret += comment
    else:
        ret = ret[:-1]

    # finish up with the outro
    ret += cm_outro
    return ret

{'fraction_non_alphanumeric': 0.17791914631643752, 'fraction_numerical': 0.002872973527601067, 'mean_word_length': 2.4814285714285713, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7469474', 'n_tokens_mistral': 1671, 'n_tokens_neox': 1595, 'n_words': 372}