<commit_before>import os
import re
import glob
import subprocess
from collections import OrderedDict
from collections.abc import Mapping
from typing import Iterator

# We need to define our own `Action` method to control the verbosity of output
# and whenever we need to run those commands in a subprocess on some platforms.
from SCons import Node
from SCons.Script import Action
from SCons.Script import ARGUMENTS
from SCons.Script import Glob
from SCons.Variables.BoolVariable import _text2bool
from platform_methods import run_in_subprocess


def add_source_files(self, sources, files):
    # Convert string to list of absolute paths (including expanding wildcard)
    if isinstance(files, (str, bytes)):
        # Keep SCons project-absolute path as they are (no wildcard support)
        if files.startswith("#"):
            if "*" in files:
                print("ERROR: Wildcards can't be expanded in SCons project-absolute path: '{}'".format(files))
                return
            files = [files]
        else:
            # Exclude .gen.cpp files from globbing, to avoid including obsolete ones.
            # They should instead be added manually.
            skip_gen_cpp = "*" in files
            dir_path = self.Dir(".").abspath
            files = sorted(glob.glob(dir_path + "/" + files))
            if skip_gen_cpp:
                files = [f for f in files if not f.endswith(".gen.cpp")]

    # Add each path as compiled Object following environment (self) configuration
    for path in files:
        obj = self.Object(path)
        if obj in sources:
            print('WARNING: Object "{}" already included in environment sources.'.format(obj))
            continue
        sources.append(obj)


def disable_warnings(self):
    # 'self' is the environment
    if self.msvc:
        # We have to remove existing warning level defines before appending /w,
        # otherwise we get: "warning D9025 : overriding '/W3' with '/w'"
        warn_flags = ["/Wall", "/W4", "/W3", "/W2", "/W1", "/WX"]
        self.Append(CCFLAGS=["/w"])
        self.Append(CFLAGS=["/w"])
        self.Append(CXXFLAGS=["/w"])
        self["CCFLAGS"] = [x for x in self["CCFLAGS"] if not x in warn_flags]
        self["CFLAGS"] = [x for x in self["CFLAGS"] if not x in warn_flags]
        self["CXXFLAGS"] = [x for x in self["CXXFLAGS"] if not x in warn_flags]
    else:
        self.Append(CCFLAGS=["-w"])
        self.Append(CFLAGS=["-w"])
        self.Append(CXXFLAGS=["-w"])


def force_optimization_on_debug(self):
    # 'self' is the environment
    if self["target"] != "debug":
        return

    if self.msvc:
        self.Append(CCFLAGS=["/O2"])
    else:
        self.Append(CCFLAGS=["-O3"])


def add_module_version_string(self, s):
    self.module_version_string += "." + s


def update_version(module_version_string=""):
    build_name = "custom_build"
    if os.getenv("BUILD_NAME") != None:
        build_name = str(os.getenv("BUILD_NAME"))
        print("Using custom build name: " + build_name)

    import version

    # NOTE: It is safe to generate this file here, since this is still executed serially
    f = open("core/version_generated.gen.h", "w")
    f.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
    f.write("#ifndef VERSION_GENERATED_GEN_H\n")
    f.write("#define VERSION_GENERATED_GEN_H\n")
    f.write('#define VERSION_SHORT_NAME "' + str(version.short_name) + '"\n')
    f.write('#define VERSION_NAME "' + str(version.name) + '"\n')
    f.write("#define VERSION_MAJOR " + str(version.major) + "\n")
    f.write("#define VERSION_MINOR " + str(version.minor) + "\n")
    f.write("#define VERSION_PATCH " + str(version.patch) + "\n")
    # For dev snapshots (alpha, beta, RC, etc.) we do not commit status change to Git,
    # so this define provides a way to override it without having to modify the source.
    godot_status = str(version.status)
    if os.getenv("GODOT_VERSION_STATUS") != None:
        godot_status = str(os.getenv("GODOT_VERSION_STATUS"))
        print("Using version status '{}', overriding the original '{}'.".format(godot_status, str(version.status)))
    f.write('#define VERSION_STATUS "' + godot_status + '"\n')
    f.write('#define VERSION_BUILD "' + str(build_name) + '"\n')
    f.write('#define VERSION_MODULE_CONFIG "' + str(version.module_config) + module_version_string + '"\n')
    f.write("#define VERSION_YEAR " + str(version.year) + "\n")
    f.write('#define VERSION_WEBSITE "' + str(version.website) + '"\n')
    f.write('#define VERSION_DOCS_BRANCH "' + str(version.docs) + '"\n')
    f.write('#define VERSION_DOCS_URL "https://docs.godotengine.org/en/" VERSION_DOCS_BRANCH\n')
    f.write("#endif // VERSION_GENERATED_GEN_H\n")
    f.close()

    # NOTE: It is safe to generate this file here, since this is still executed serially
    fhash = open("core/version_hash.gen.cpp", "w")
    fhash.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
    fhash.write('#include "core/version.h"\n')
    githash = ""
    gitfolder = ".git"

    if os.path.isfile(".git"):
        module_folder = open(".git", "r").readline().strip()
        if module_folder.startswith("gitdir: "):
            gitfolder = module_folder[8:]

    if os.path.isfile(os.path.join(gitfolder, "HEAD")):
        head = open(os.path.join(gitfolder, "HEAD"), "r", encoding="utf8").readline().strip()
        if head.startswith("ref: "):
            head = os.path.join(gitfolder, head[5:])
            if os.path.isfile(head):
                githash = open(head, "r").readline().strip()<commit_msg>support git packed refs in version generator<commit_after>import os
import re
import glob
import subprocess
from collections import OrderedDict
from collections.abc import Mapping
from typing import Iterator

# We need to define our own `Action` method to control the verbosity of output
# and whenever we need to run those commands in a subprocess on some platforms.
from SCons import Node
from SCons.Script import Action
from SCons.Script import ARGUMENTS
from SCons.Script import Glob
from SCons.Variables.BoolVariable import _text2bool
from platform_methods import run_in_subprocess


def add_source_files(self, sources, files):
    # Convert string to list of absolute paths (including expanding wildcard)
    if isinstance(files, (str, bytes)):
        # Keep SCons project-absolute path as they are (no wildcard support)
        if files.startswith("#"):
            if "*" in files:
                print("ERROR: Wildcards can't be expanded in SCons project-absolute path: '{}'".format(files))
                return
            files = [files]
        else:
            # Exclude .gen.cpp files from globbing, to avoid including obsolete ones.
            # They should instead be added manually.
            skip_gen_cpp = "*" in files
            dir_path = self.Dir(".").abspath
            files = sorted(glob.glob(dir_path + "/" + files))
            if skip_gen_cpp:
                files = [f for f in files if not f.endswith(".gen.cpp")]

    # Add each path as compiled Object following environment (self) configuration
    for path in files:
        obj = self.Object(path)
        if obj in sources:
            print('WARNING: Object "{}" already included in environment sources.'.format(obj))
            continue
        sources.append(obj)


def disable_warnings(self):
    # 'self' is the environment
    if self.msvc:
        # We have to remove existing warning level defines before appending /w,
        # otherwise we get: "warning D9025 : overriding '/W3' with '/w'"
        warn_flags = ["/Wall", "/W4", "/W3", "/W2", "/W1", "/WX"]
        self.Append(CCFLAGS=["/w"])
        self.Append(CFLAGS=["/w"])
        self.Append(CXXFLAGS=["/w"])
        self["CCFLAGS"] = [x for x in self["CCFLAGS"] if not x in warn_flags]
        self["CFLAGS"] = [x for x in self["CFLAGS"] if not x in warn_flags]
        self["CXXFLAGS"] = [x for x in self["CXXFLAGS"] if not x in warn_flags]
    else:
        self.Append(CCFLAGS=["-w"])
        self.Append(CFLAGS=["-w"])
        self.Append(CXXFLAGS=["-w"])


def force_optimization_on_debug(self):
    # 'self' is the environment
    if self["target"] != "debug":
        return

    if self.msvc:
        self.Append(CCFLAGS=["/O2"])
    else:
        self.Append(CCFLAGS=["-O3"])


def add_module_version_string(self, s):
    self.module_version_string += "." + s


def update_version(module_version_string=""):
    build_name = "custom_build"
    if os.getenv("BUILD_NAME") != None:
        build_name = str(os.getenv("BUILD_NAME"))
        print("Using custom build name: " + build_name)

    import version

    # NOTE: It is safe to generate this file here, since this is still executed serially
    f = open("core/version_generated.gen.h", "w")
    f.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
    f.write("#ifndef VERSION_GENERATED_GEN_H\n")
    f.write("#define VERSION_GENERATED_GEN_H\n")
    f.write('#define VERSION_SHORT_NAME "' + str(version.short_name) + '"\n')
    f.write('#define VERSION_NAME "' + str(version.name) + '"\n')
    f.write("#define VERSION_MAJOR " + str(version.major) + "\n")
    f.write("#define VERSION_MINOR " + str(version.minor) + "\n")
    f.write("#define VERSION_PATCH " + str(version.patch) + "\n")
    # For dev snapshots (alpha, beta, RC, etc.) we do not commit status change to Git,
    # so this define provides a way to override it without having to modify the source.
    godot_status = str(version.status)
    if os.getenv("GODOT_VERSION_STATUS") != None:
        godot_status = str(os.getenv("GODOT_VERSION_STATUS"))
        print("Using version status '{}', overriding the original '{}'.".format(godot_status, str(version.status)))
    f.write('#define VERSION_STATUS "' + godot_status + '"\n')
    f.write('#define VERSION_BUILD "' + str(build_name) + '"\n')
    f.write('#define VERSION_MODULE_CONFIG "' + str(version.module_config) + module_version_string + '"\n')
    f.write("#define VERSION_YEAR " + str(version.year) + "\n")
    f.write('#define VERSION_WEBSITE "' + str(version.website) + '"\n')
    f.write('#define VERSION_DOCS_BRANCH "' + str(version.docs) + '"\n')
    f.write('#define VERSION_DOCS_URL "https://docs.godotengine.org/en/" VERSION_DOCS_BRANCH\n')
    f.write("#endif // VERSION_GENERATED_GEN_H\n")
    f.close()

    # NOTE: It is safe to generate this file here, since this is still executed serially
    fhash = open("core/version_hash.gen.cpp", "w")
    fhash.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
    fhash.write('#include "core/version.h"\n')
    githash = ""
    gitfolder = ".git"

    if os.path.isfile(".git"):
        module_folder = open(".git", "r").readline().strip()
        if module_folder.startswith("gitdir: "):
            gitfolder = module_folder[8:]

    if os.path.isfile(os.path.join(gitfolder, "HEAD")):
        head = open(os.path.join(gitfolder, "HEAD"), "r", encoding="utf8").readline().strip()
        if head.startswith("ref: "):
            ref = head[5:]
            head = os.path.join(gitfolder, ref)
            packedrefs = os.path.join(gitfolder, "packed-refs")
            if os.path.isfile(head):
                githash = open(head, "r").readline().strip()
            elif os.path.isfile(packedrefs):
                # Git may pack refs into a single file. This code searches .git/packed-refs file for the current ref's hash.
                # https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-pack-refs.html
                for line in open(packedrefs, "r").read().splitlines():
                    if line.startswith("#"):
                        continue
                    (line_hash, line_ref) = line.split(" ")
                    if ref == line_ref:
                        githash = line_hash
                        break
{'fraction_non_alphanumeric': 0.13630584046791558, 'fraction_numerical': 0.002543019411714843, 'mean_word_length': 2.8505221932114884, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3539484', 'n_tokens_mistral': 3697, 'n_tokens_neox': 3632, 'n_words': 1010}