<commit_before>#!/usr/bin/python3
"""
Command line utility to generate compile_commands.json for RIOT applications
"""
import argparse
import json
import os
import re
import shlex
import subprocess
import sys

REGEX_VERSION = re.compile(r"\ngcc version ([^ ]+)")
REGEX_INCLUDES = r"^#include <\.\.\.> search starts here:$((?:\n|\r|.)*?)^End of search list\.$"
REGEX_INCLUDES = re.compile(REGEX_INCLUDES, re.MULTILINE)


def detect_includes_and_version_gcc(compiler):
    """
    Runs the given compiler with -v -E on an no-op compilation unit and parses the built-in
    include search directories and the GCC version from the output

    :param compiler: name / path of the compiler to run
    :type compiler: str

    :return: (list_of_include_paths, version)
    :rtype: tuple
    """
    try:
        with subprocess.Popen([compiler, "-v", "-E", "-"],
                              stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE) as proc:
            inputdata = b"typedef int dont_be_pedantic;"
            _, stderrdata = proc.communicate(input=inputdata)
    except FileNotFoundError:
        msg = "Compiler {} not found, not adding system include paths\n".format(compiler)
        sys.stderr.write(msg)
        return []

    stderrdata = stderrdata.decode("utf-8")
    version = REGEX_VERSION.search(stderrdata).group(1)
    includes = [os.path.abspath(p) for p in REGEX_INCLUDES.search(stderrdata).group(1).split()]

    return (includes, version)


def detect_libstdcxx_includes(compiler, includes, version):
    """
    Tries to detect the g++ libstdc++ built-in include search directories using black magic and
    adds them to the list given in includes

    :param compiler: Name or path of the compiler
    :type compiler: str
    :param includes: List of include directories
    :type includes: list of str
    :param version: Version of g++
    :type version: str
    """
    for path in includes:
        cxx_lib = os.path.join(path, "c++", version)
        if os.path.exists(cxx_lib):
            includes.append(cxx_lib)
            triple = os.path.basename(compiler)[0:-4]
            cxx_extra = os.path.join(cxx_lib, triple)
            if os.path.exists(cxx_extra):
                includes.append(cxx_extra)
            break


def detect_built_in_includes(compiler, args):
    """
    Tries to detect the built-in include search directories of the given compiler

    :param compiler: Name or path of the compiler
    :type compiler: str
    :param args: Command line arguments

    :return: List of built-in include directories
    :rtype: list of str
    """
    if compiler.endswith('-gcc'):
        includes, version = detect_includes_and_version_gcc(compiler)
    elif compiler.endswith('-g++'):
        includes, version = detect_includes_and_version_gcc(compiler)
        if args.add_libstdcxx_includes:
            detect_libstdcxx_includes(compiler, includes, version)
    elif compiler in ('clang', 'clang++', 'gcc', 'g++'):
        # clang / clang++ doesn't have any magic include search dirs built in, so we don't need
        # to detect them.
        # for host gcc/g++ we don't need to detect magic include dirs either.
        includes = []
    else:
        msg = "Warning: Cannot detect default include search paths for {}\n".format(compiler)
        sys.stderr.write(msg)
        includes = []
    return includes


class CompilationDetails:  # pylint: disable=too-few-public-methods,too-many-instance-attributes
    """
    Representation of the compilation details stored by RIOT's build system.

    :param path: Path of the module to read compilation details from
    :type path: str
    """
    def __init__(self, path):  # pylint: disable=too-many-branches
        with open(os.path.join(path, "compile_cmds.txt"), "r") as file:
            for line in file.read().splitlines():
                if line.startswith("SRC: "):
                    self.src_c = line.lstrip("SRC: ").split()
                elif line.startswith("SRC_NO_LTO: "):
                    self.src_c_no_lto = line.lstrip("SRC_NO_LTO: ").split()
                elif line.startswith("SRCXX: "):
                    self.src_cxx = line.lstrip("SRCXX: ").split()
                elif line.startswith("CURDIR: "):
                    self.dir = line.lstrip("CURDIR: ").strip()
                elif line.startswith("CFLAGS: "):
                    self.cflags = shlex.split(line.lstrip("CFLAGS: "))
                elif line.startswith("LTOFLAGS: "):
                    self.ltoflags = shlex.split(line.lstrip("LTOFLAGS: "))
                elif line.startswith("INCLUDES: "):
                    self.includes = shlex.split(line.lstrip("INCLUDES: "))
                elif line.startswith("CXXFLAGS: "):
                    self.cxxflags = shlex.split(line.lstrip("CXXFLAGS: "))
                elif line.startswith("CXXINCLUDES: "):
                    self.cxxincludes = shlex.split(line.lstrip("CXXINCLUDES: "))
                elif line.startswith("CC: "):
                    self.cc = line.lstrip("CC: ").strip()  # pylint: disable=invalid-name
                elif line.startswith("CXX: "):
                    self.cxx = line.lstrip("CXX: ").strip()
                elif line.startswith("TARGET_ARCH: "):
                    self.target_arch = line.lstrip("TARGET_ARCH: ").strip()
                elif line.startswith("TARGET_ARCH_LLVM: "):
                    self.target_arch_llvm = line.lstrip("TARGET_ARCH_LLVM: ").strip()


class State:  # pylint: disable=too-few-public-methods
    """
    Entity to store the current programs state
    """
    def __init__(self):
        self.def_includes = dict()
        self.is_first = True


def get_built_in_include_flags(compiler, state, args):
    """
    Get built-in include search directories as parameter list.

    :param compiler: Name or path of the compiler to get the include search dirs from
    :type compiler: str
    :param state: state of the program
    :param args: command line arguments

    :return: The -isystem <...> compiler flags for the built-in include search dirs as list
    :rtype: list
    """

    result = []
    if compiler not in state.def_includes:
        state.def_includes[compiler] = detect_built_in_includes(compiler, args)

    for include in state.def_includes[compiler]:
        result.append('-isystem')
        result.append(include)

    return result


def write_compile_command(state, compiler, src, flags, cdetails, path):
    """
    Write the compile command for the given source file with the given parameters to stdout

    :param state: state of the program
    :param compiler: the C/C++ compiler used
    :type compiler: str
    :param src: the file to compiler
    :type src: str
    :param flags: flags used for compiler invocation
    :type flags: list of str
    :param cetails: compilation details
    :type cdetails: CompilationDetails
    :param path: the output path
    "type path: str
    """
    if state.is_first:
        state.is_first = False
    else:
        sys.stdout.write(",\n")
    obj = os.path.splitext(src)[0] + ".o"
    arguments = [compiler, '-DRIOT_FILE_RELATIVE="' + os.path.join(cdetails.dir, src) + '"',
                 '-DRIOT_FILE_NOPATH="' + src + '"']
    arguments += flags<commit_msg>compile_commands: Don't produce duplicate -c arguments<commit_after>#!/usr/bin/python3
"""
Command line utility to generate compile_commands.json for RIOT applications
"""
import argparse
import json
import os
import re
import shlex
import subprocess
import sys

REGEX_VERSION = re.compile(r"\ngcc version ([^ ]+)")
REGEX_INCLUDES = r"^#include <\.\.\.> search starts here:$((?:\n|\r|.)*?)^End of search list\.$"
REGEX_INCLUDES = re.compile(REGEX_INCLUDES, re.MULTILINE)


def detect_includes_and_version_gcc(compiler):
    """
    Runs the given compiler with -v -E on an no-op compilation unit and parses the built-in
    include search directories and the GCC version from the output

    :param compiler: name / path of the compiler to run
    :type compiler: str

    :return: (list_of_include_paths, version)
    :rtype: tuple
    """
    try:
        with subprocess.Popen([compiler, "-v", "-E", "-"],
                              stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE) as proc:
            inputdata = b"typedef int dont_be_pedantic;"
            _, stderrdata = proc.communicate(input=inputdata)
    except FileNotFoundError:
        msg = "Compiler {} not found, not adding system include paths\n".format(compiler)
        sys.stderr.write(msg)
        return []

    stderrdata = stderrdata.decode("utf-8")
    version = REGEX_VERSION.search(stderrdata).group(1)
    includes = [os.path.abspath(p) for p in REGEX_INCLUDES.search(stderrdata).group(1).split()]

    return (includes, version)


def detect_libstdcxx_includes(compiler, includes, version):
    """
    Tries to detect the g++ libstdc++ built-in include search directories using black magic and
    adds them to the list given in includes

    :param compiler: Name or path of the compiler
    :type compiler: str
    :param includes: List of include directories
    :type includes: list of str
    :param version: Version of g++
    :type version: str
    """
    for path in includes:
        cxx_lib = os.path.join(path, "c++", version)
        if os.path.exists(cxx_lib):
            includes.append(cxx_lib)
            triple = os.path.basename(compiler)[0:-4]
            cxx_extra = os.path.join(cxx_lib, triple)
            if os.path.exists(cxx_extra):
                includes.append(cxx_extra)
            break


def detect_built_in_includes(compiler, args):
    """
    Tries to detect the built-in include search directories of the given compiler

    :param compiler: Name or path of the compiler
    :type compiler: str
    :param args: Command line arguments

    :return: List of built-in include directories
    :rtype: list of str
    """
    if compiler.endswith('-gcc'):
        includes, version = detect_includes_and_version_gcc(compiler)
    elif compiler.endswith('-g++'):
        includes, version = detect_includes_and_version_gcc(compiler)
        if args.add_libstdcxx_includes:
            detect_libstdcxx_includes(compiler, includes, version)
    elif compiler in ('clang', 'clang++', 'gcc', 'g++'):
        # clang / clang++ doesn't have any magic include search dirs built in, so we don't need
        # to detect them.
        # for host gcc/g++ we don't need to detect magic include dirs either.
        includes = []
    else:
        msg = "Warning: Cannot detect default include search paths for {}\n".format(compiler)
        sys.stderr.write(msg)
        includes = []
    return includes


class CompilationDetails:  # pylint: disable=too-few-public-methods,too-many-instance-attributes
    """
    Representation of the compilation details stored by RIOT's build system.

    :param path: Path of the module to read compilation details from
    :type path: str
    """
    def __init__(self, path):  # pylint: disable=too-many-branches
        with open(os.path.join(path, "compile_cmds.txt"), "r") as file:
            for line in file.read().splitlines():
                if line.startswith("SRC: "):
                    self.src_c = line.lstrip("SRC: ").split()
                elif line.startswith("SRC_NO_LTO: "):
                    self.src_c_no_lto = line.lstrip("SRC_NO_LTO: ").split()
                elif line.startswith("SRCXX: "):
                    self.src_cxx = line.lstrip("SRCXX: ").split()
                elif line.startswith("CURDIR: "):
                    self.dir = line.lstrip("CURDIR: ").strip()
                elif line.startswith("CFLAGS: "):
                    self.cflags = shlex.split(line.lstrip("CFLAGS: "))
                elif line.startswith("LTOFLAGS: "):
                    self.ltoflags = shlex.split(line.lstrip("LTOFLAGS: "))
                elif line.startswith("INCLUDES: "):
                    self.includes = shlex.split(line.lstrip("INCLUDES: "))
                elif line.startswith("CXXFLAGS: "):
                    self.cxxflags = shlex.split(line.lstrip("CXXFLAGS: "))
                elif line.startswith("CXXINCLUDES: "):
                    self.cxxincludes = shlex.split(line.lstrip("CXXINCLUDES: "))
                elif line.startswith("CC: "):
                    self.cc = line.lstrip("CC: ").strip()  # pylint: disable=invalid-name
                elif line.startswith("CXX: "):
                    self.cxx = line.lstrip("CXX: ").strip()
                elif line.startswith("TARGET_ARCH: "):
                    self.target_arch = line.lstrip("TARGET_ARCH: ").strip()
                elif line.startswith("TARGET_ARCH_LLVM: "):
                    self.target_arch_llvm = line.lstrip("TARGET_ARCH_LLVM: ").strip()


class State:  # pylint: disable=too-few-public-methods
    """
    Entity to store the current programs state
    """
    def __init__(self):
        self.def_includes = dict()
        self.is_first = True


def get_built_in_include_flags(compiler, state, args):
    """
    Get built-in include search directories as parameter list.

    :param compiler: Name or path of the compiler to get the include search dirs from
    :type compiler: str
    :param state: state of the program
    :param args: command line arguments

    :return: The -isystem <...> compiler flags for the built-in include search dirs as list
    :rtype: list
    """

    result = []
    if compiler not in state.def_includes:
        state.def_includes[compiler] = detect_built_in_includes(compiler, args)

    for include in state.def_includes[compiler]:
        result.append('-isystem')
        result.append(include)

    return result


def write_compile_command(state, compiler, src, flags, cdetails, path):
    """
    Write the compile command for the given source file with the given parameters to stdout

    :param state: state of the program
    :param compiler: the C/C++ compiler used
    :type compiler: str
    :param src: the file to compiler
    :type src: str
    :param flags: flags used for compiler invocation
    :type flags: list of str
    :param cetails: compilation details
    :type cdetails: CompilationDetails
    :param path: the output path
    "type path: str
    """
    if state.is_first:
        state.is_first = False
    else:
        sys.stdout.write(",\n")
    obj = os.path.splitext(src)[0] + ".o"
    arguments = [compiler, '-DRIOT_FILE_RELATIVE="' + os.path.join(cdetails.dir, src) + '"',
                 '-DRIOT_FILE_NOPATH="' + src + '"']
    arguments += flags
    if '-c' in arguments:
        # bindgen is unhappy with multiple -c (that would be created by the -c
        # added later) and even with the -c showing up anywhere between other
        # arguments.
        assert arguments.count('-c') == 1, "Spurious duplicate -c arguments"
        arguments.remove('-c')
{'fraction_non_alphanumeric': 0.11239558970932174, 'fraction_numerical': 0.0010023387905111927, 'mean_word_length': 2.509026963657679, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4884314', 'n_tokens_mistral': 4548, 'n_tokens_neox': 4351, 'n_words': 1236}