<commit_before>from scmtools.core import *
import re

class PerforceTool(SCMTool):
    def __init__(self, repopath):
        SCMTool.__init__(self, repopath)

    @staticmethod
    def parse_change_desc(changenum, changedesc):
        changeset = ChangeSet()
        changeset.changenum = changenum

        changedesc_keys = {
            'QA Notes': "",
            'Testing Done': "",
            'Documentation Notes': "",
            'Bug Number': "",
            'Reviewed by': "",
            'Approved by': "",
            'Breaks vmcore compatibility': "",
            'Breaks vmkernel compatibility': "",
            'Breaks vmkdrivers compatibility': "",
            'Mailto': "",
        }

        process_summary = False
        process_description = False
        process_files = False

        cur_key = None

        for line in changedesc.split("\n"):
            if line == "Description:":
                process_summary = True
                continue
            elif line == "Files:":
                process_files = True
                cur_key = None
                continue
            elif line.strip() == "":
                if process_summary:
                    process_summary = False
                    process_description = True
                    continue

                line = ""
            elif line.startswith("\t") or line.startswith("    "):
                line = line.lstrip()

                if process_files:
                    changeset.files.append(line)
                    continue
                elif line.find(':') != -1:
                    key, value = line.split(':', 2)

                    if changedesc_keys.has_key(key):
                        process_description = False
                        cur_key = key

                        changedesc_keys[key] = value.lstrip() + "\n"
                        continue

            line += "\n"

            if process_summary:
                changeset.summary += line
            elif process_description:
                changeset.description += line
            elif cur_key != None:
                changedesc_keys[cur_key] += line

        changeset.summary = changeset.summary.strip()
        changeset.description = changeset.description.strip()
        changeset.testing_done = changedesc_keys['Testing Done'].strip()
        changeset.bugs_closed = re.split(r"[, ]+",
                                         changedesc_keys['Bug Number'])

        # This is gross.
        if len(changeset.files) > 0:
            parts = changeset.files[0].split('/')

            if parts[2] == "depot":
                changeset.branch = parts[4]

        return changeset<commit_msg>Strip the list of bugs so we don't get trailing whitespace on the last bug number.<commit_after>from scmtools.core import *
import re

class PerforceTool(SCMTool):
    def __init__(self, repopath):
        SCMTool.__init__(self, repopath)

    @staticmethod
    def parse_change_desc(changenum, changedesc):
        changeset = ChangeSet()
        changeset.changenum = changenum

        changedesc_keys = {
            'QA Notes': "",
            'Testing Done': "",
            'Documentation Notes': "",
            'Bug Number': "",
            'Reviewed by': "",
            'Approved by': "",
            'Breaks vmcore compatibility': "",
            'Breaks vmkernel compatibility': "",
            'Breaks vmkdrivers compatibility': "",
            'Mailto': "",
        }

        process_summary = False
        process_description = False
        process_files = False

        cur_key = None

        for line in changedesc.split("\n"):
            if line == "Description:":
                process_summary = True
                continue
            elif line == "Files:":
                process_files = True
                cur_key = None
                continue
            elif line.strip() == "":
                if process_summary:
                    process_summary = False
                    process_description = True
                    continue

                line = ""
            elif line.startswith("\t") or line.startswith("    "):
                line = line.lstrip()

                if process_files:
                    changeset.files.append(line)
                    continue
                elif line.find(':') != -1:
                    key, value = line.split(':', 2)

                    if changedesc_keys.has_key(key):
                        process_description = False
                        cur_key = key

                        changedesc_keys[key] = value.lstrip() + "\n"
                        continue

            line += "\n"

            if process_summary:
                changeset.summary += line
            elif process_description:
                changeset.description += line
            elif cur_key != None:
                changedesc_keys[cur_key] += line

        changeset.summary = changeset.summary.strip()
        changeset.description = changeset.description.strip()
        changeset.testing_done = changedesc_keys['Testing Done'].strip()
        changeset.bugs_closed = re.split(r"[, ]+",
                                         changedesc_keys['Bug Number'].strip())

        # This is gross.
        if len(changeset.files) > 0:
            parts = changeset.files[0].split('/')

            if parts[2] == "depot":
                changeset.branch = parts[4]

        return changeset
{'fraction_non_alphanumeric': 0.09979857169016663, 'fraction_numerical': 0.0021973997436366965, 'mean_word_length': 1.3665511265164645, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6397183', 'n_tokens_mistral': 1395, 'n_tokens_neox': 1298, 'n_words': 308}