<commit_before>#!/usr/bin/env python

import argparse
import re
import subprocess

from enum import Enum

EXCLUDED_SUBJECTS = {
    "AutoValue",
    "avadoc",
    "avaDoc",
    "ava-doc",
    "baz",  # bazel, bazlet(s)
    "Baz",
    "class",
    "efactor",
    "format",
    "Format",
    "getter",
    "gr-",
    "immutab",
    "IT",
    "js",
    "lint",
    "method",
    "module",
    "naming",
    "nits",
    "nongoogle",
    "prone",  # error prone &co.
    "register",
    "Register",
    "remove",
    "Remove",
    "rename",
    "Rename",
    "Revert",
    "serializ",
    "setter",
    "spell",
    "Spell",
    "test",  # testing, tests; unit or else
    "Test",
    "thread",
    "tsetse",
    "typescript",
    "version",
}

COMMIT_SHA1_PATTERN = r"^commit ([a-z0-9]+)$"
DATE_HEADER_PATTERN = r"Date: .+"
SUBJECT_SUBMODULES_PATTERN = r"^Update git submodules$"
UPDATE_SUBMODULE_PATTERN = r"\* Update ([a-z/\-]+) from branch '.+'"
SUBMODULE_SUBJECT_PATTERN = r"^- (.+)"
SUBMODULE_MERGE_PATTERN = r".+Merge .+"
ISSUE_ID_PATTERN = r"[a-zA-Z]+: [Ii]ssue ([0-9]+)"
CHANGE_ID_PATTERN = r"^Change-Id: [I0-9a-z]+$"
PLUGIN_PATTERN = r"plugins/([a-z\-]+)"
RELEASE_OPTION_PATTERN = r".+\.\.(v.+)"
RELEASE_TAG_PATTERN = r"v[0-9]+\.[0-9]+\.[0-9]+$"

ISSUE_URL = "https://bugs.chromium.org/p/gerrit/issues/detail?id="
CHECK_DISCLAIMER = "experimental and much slower"
GIT_COMMAND = "git"
UTF8 = "UTF-8"


def parse_args():
    parser = argparse.ArgumentParser(
        description="Generate an initial release notes markdown file.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser.add_argument(
        "-c",
        "--check",
        dest="check",
        required=False,
        default=False,
        action="store_true",
        help=f"check commits for previous releases; {CHECK_DISCLAIMER}",
    )
    parser.add_argument("range", help="git log revision range")
    return parser.parse_args()


def check_args(options):
    if not options.check:
        return None
    release_option = re.search(RELEASE_OPTION_PATTERN, options.range)
    if release_option is None:
        print("Check option ignored; range doesn't end with release tag.")
        return None
    print(f"Check option used; {CHECK_DISCLAIMER}.")
    return release_option.group(1)


def newly_released(commit_sha1, release):
    if release is None:
        return True
    git_tag = [
        GIT_COMMAND,
        "tag",
        "--contains",
        commit_sha1,
    ]
    process = subprocess.Popen(git_tag, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    verdict = True
    for line in iter(process.stdout.readline, ""):
        if process.poll() is not None:
            break
        line = line.strip().decode(UTF8)
        if not re.match(rf"{re.escape(release)}$", line):
            # Wrongfully pushed or malformed tags ignored.
            # Preceding release-candidate (-rcN) tags treated as newly released.
            verdict = not re.match(RELEASE_TAG_PATTERN, line)
    return verdict


def open_git_log(options):
    git_log = [
        GIT_COMMAND,
        "log",
        "--no-merges",
        options.range,
    ]
    return subprocess.Popen(git_log, stdout=subprocess.PIPE)


class Change:
    subject = None
    issues = set()


class Task(Enum):
    start_commit = 1
    finish_headers = 2
    capture_subject = 3
    capture_submodule = 4
    capture_submodule_subject = 5
    finish_submodule_change = 6
    finish_commit = 7


class Commit:
    sha1 = None
    subject = None
    submodule = None
    issues = set()

    def reset(self, signature, task):
        if signature is not None:
            self.sha1 = signature.group(1)
            self.subject = None
            self.submodule = None
            self.issues = set()
            return Task.finish_headers
        return task


def parse_log(process, release):
    commit = Commit()
    commits = []
    submodules = dict()
    submodule_change = None
    task = Task.start_commit
    for line in iter(process.stdout.readline, ""):
        if process.poll() is not None:
            break
        line = line.strip().decode(UTF8)
        if not line:
            continue
        if task == Task.start_commit:
            task = commit.reset(re.search(COMMIT_SHA1_PATTERN, line), task)
        elif task == Task.finish_headers:
            if re.match(DATE_HEADER_PATTERN, line):
                task = Task.capture_subject
        elif task == Task.capture_subject:
            if re.match(SUBJECT_SUBMODULES_PATTERN, line):
                task = Task.capture_submodule
            else:
                commit.subject = line
                task = Task.finish_commit
        elif task == Task.capture_submodule:
            commit.submodule = re.search(UPDATE_SUBMODULE_PATTERN, line).group(1)
            if commit.submodule not in submodules:
                submodules[commit.submodule] = []
            task = Task.capture_submodule_subject
        elif task == Task.capture_submodule_subject:
            submodule_subject = re.search(SUBMODULE_SUBJECT_PATTERN, line)<commit_msg>Replace flaky Popen with subprocess.check_output in release_noter<commit_after>#!/usr/bin/env python

import argparse
import re
import subprocess

from enum import Enum

EXCLUDED_SUBJECTS = {
    "AutoValue",
    "avadoc",
    "avaDoc",
    "ava-doc",
    "baz",  # bazel, bazlet(s)
    "Baz",
    "class",
    "efactor",
    "format",
    "Format",
    "getter",
    "gr-",
    "immutab",
    "IT",
    "js",
    "lint",
    "method",
    "module",
    "naming",
    "nits",
    "nongoogle",
    "prone",  # error prone &co.
    "register",
    "Register",
    "remove",
    "Remove",
    "rename",
    "Rename",
    "Revert",
    "serializ",
    "setter",
    "spell",
    "Spell",
    "test",  # testing, tests; unit or else
    "Test",
    "thread",
    "tsetse",
    "typescript",
    "version",
}

COMMIT_SHA1_PATTERN = r"^commit ([a-z0-9]+)$"
DATE_HEADER_PATTERN = r"Date: .+"
SUBJECT_SUBMODULES_PATTERN = r"^Update git submodules$"
UPDATE_SUBMODULE_PATTERN = r"\* Update ([a-z/\-]+) from branch '.+'"
SUBMODULE_SUBJECT_PATTERN = r"^- (.+)"
SUBMODULE_MERGE_PATTERN = r".+Merge .+"
ISSUE_ID_PATTERN = r"[a-zA-Z]+: [Ii]ssue ([0-9]+)"
CHANGE_ID_PATTERN = r"^Change-Id: [I0-9a-z]+$"
PLUGIN_PATTERN = r"plugins/([a-z\-]+)"
RELEASE_OPTION_PATTERN = r".+\.\.(v.+)"
RELEASE_TAG_PATTERN = r"v[0-9]+\.[0-9]+\.[0-9]+$"

ISSUE_URL = "https://bugs.chromium.org/p/gerrit/issues/detail?id="
CHECK_DISCLAIMER = "experimental and much slower"
GIT_COMMAND = "git"
UTF8 = "UTF-8"


def parse_args():
    parser = argparse.ArgumentParser(
        description="Generate an initial release notes markdown file.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser.add_argument(
        "-c",
        "--check",
        dest="check",
        required=False,
        default=False,
        action="store_true",
        help=f"check commits for previous releases; {CHECK_DISCLAIMER}",
    )
    parser.add_argument("range", help="git log revision range")
    return parser.parse_args()


def check_args(options):
    if not options.check:
        return None
    release_option = re.search(RELEASE_OPTION_PATTERN, options.range)
    if release_option is None:
        print("Check option ignored; range doesn't end with release tag.")
        return None
    print(f"Check option used; {CHECK_DISCLAIMER}.")
    return release_option.group(1)


def newly_released(commit_sha1, release):
    if release is None:
        return True
    git_tag = [
        GIT_COMMAND,
        "tag",
        "--contains",
        commit_sha1,
    ]
    process = subprocess.check_output(git_tag, stderr=subprocess.PIPE, encoding=UTF8)
    verdict = True
    for line in process.splitlines():
        line = line.strip()
        if not re.match(rf"{re.escape(release)}$", line):
            # Wrongfully pushed or malformed tags ignored.
            # Preceding release-candidate (-rcN) tags treated as newly released.
            verdict = not re.match(RELEASE_TAG_PATTERN, line)
    return verdict


def open_git_log(options):
    git_log = [
        GIT_COMMAND,
        "log",
        "--no-merges",
        options.range,
    ]
    return subprocess.check_output(git_log, encoding=UTF8)


class Change:
    subject = None
    issues = set()


class Task(Enum):
    start_commit = 1
    finish_headers = 2
    capture_subject = 3
    capture_submodule = 4
    capture_submodule_subject = 5
    finish_submodule_change = 6
    finish_commit = 7


class Commit:
    sha1 = None
    subject = None
    submodule = None
    issues = set()

    def reset(self, signature, task):
        if signature is not None:
            self.sha1 = signature.group(1)
            self.subject = None
            self.submodule = None
            self.issues = set()
            return Task.finish_headers
        return task


def parse_log(process, release):
    commit = Commit()
    commits = []
    submodules = dict()
    submodule_change = None
    task = Task.start_commit
    for line in process.splitlines():
        line = line.strip()
        if not line:
            continue
        if task == Task.start_commit:
            task = commit.reset(re.search(COMMIT_SHA1_PATTERN, line), task)
        elif task == Task.finish_headers:
            if re.match(DATE_HEADER_PATTERN, line):
                task = Task.capture_subject
        elif task == Task.capture_subject:
            if re.match(SUBJECT_SUBMODULES_PATTERN, line):
                task = Task.capture_submodule
            else:
                commit.subject = line
                task = Task.finish_commit
        elif task == Task.capture_submodule:
            commit.submodule = re.search(UPDATE_SUBMODULE_PATTERN, line).group(1)
            if commit.submodule not in submodules:
                submodules[commit.submodule] = []
            task = Task.capture_submodule_subject
        elif task == Task.capture_submodule_subject:
            submodule_subject = re.search(SUBMODULE_SUBJECT_PATTERN, line)
{'fraction_non_alphanumeric': 0.12313655336911151, 'fraction_numerical': 0.006360564500099384, 'mean_word_length': 2.4391660970608338, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3088414', 'n_tokens_mistral': 3365, 'n_tokens_neox': 3251, 'n_words': 719}