<commit_before>#!/usr/bin/env python
# Copyright 2016-present Barefoot Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script examines the source tree, starting at the current directory, and
# automatically generates an automake file which can be included by Makefile.am
# to enable unified compilation. The idea of unified compilation is to create a
# "unified" C++ source file which simply #include's other source files.  Rather
# than compiling the original source files, we compile only the unified file,
# which greatly reduces compilation overhead in terms of I/O, duplicate template
# instantiations, and the like. This makes compiling the project much faster.
#
# Each automake file (i.e., file with '.am' extension) in the source tree is
# examined to find any variables of the form 'foo_UNIFIED' and 'foo_NONUNIFIED'.
# The script generates makefile rules that create a '.cpp' file that #include's
# the 'foo_UNIFIED' files. The '.cpp' file is added to 'foo_SOURCES' so that it
# will be compiled, and the usual automake features like dependency tracking
# work as expected. 'foo_NONUNIFIED' files are added directly to 'foo_SOURCES'
# and continue to be compiled separately.
#
# Putting every 'foo_UNIFIED' file in the same generated '.cpp' file would
# maximize single-threaded compilation speed, but it would limit parallelism and
# slow down incremental builds. This is why the 'foo_UNIFIED' files are split
# into chunks, with each chunk getting its own '.cpp' file which is compiled
# separately. The size of the chunks is a tunable parameter; if the default
# value isn't working well, it can be overridden by providing a custom chunk
# size as a parameter to this script.
#
# Correct usage requires setting up the makefiles appropriately. Since this
# script manages 'foo_SOURCES' itself, specifying 'foo_SOURCES' is an error if
# 'foo_UNIFIED' or 'foo_NONUNIFIED' is specified. This helps avoid unexpected
# interactions. 'foo_UNIFIED' also can't include any expansions ("$FOO" or
# "$(foo)") or any other makefile trickery, and it can't include file types
# other than '.c' and '.cpp' files. That includes headers; if you really want to
# include them somewhere, 'foo_NONUNIFIED' is the place, but the best thing is
# to leave them out altogether and let automake's automatic dependency
# management handle them.
#
# It's not necessary to worry about excluding automake files that aren't
# relevant to the target you're building. This script only generates make rules;
# if you're not building a target that triggers them, they won't have any
# effect.
#
# The output of the script is written to stdout, so all that's necessary to use
# it is to run it from the root of the source tree before automake is run,
# redirect the output to a file, and then include that file from your
# Makefile.am. The script requires that Makefile.am also set a GEN_UNIFIED_CPP
# variable which points to the script that will actually create the generated
# '.cpp' files at build time. (That's gen-unified-sources.py.) Doing it this way
# allows this tool to run before the build directory has been created, and
# avoids generating '.cpp' files that aren't needed by the targets we're
# building.

from __future__ import print_function
import collections
import fnmatch
import math
import os
import re
import stat
import sys

def find_automake_files():
    files = []
    for root, _, filenames in os.walk('.', symlinks=True):
        for filename in fnmatch.filter(filenames, '*.am'):
            files.append(os.path.join(root, filename))
    return files

def construct_file_set(makefile, reldir, target, type, files):
    for file in files:
        # Filter empty filenames. These end up getting passed to us because of
        # split()'s behavior when the regular expression contains a capture
        # group.
        if file == '':
            continue

        # Replace %reldir% replaced with a concrete path.
        file = file.replace('%reldir%', reldir)<commit_msg>use followlinks instead of symlinks in tools/gen-unified-makefile.py<commit_after>#!/usr/bin/env python
# Copyright 2016-present Barefoot Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script examines the source tree, starting at the current directory, and
# automatically generates an automake file which can be included by Makefile.am
# to enable unified compilation. The idea of unified compilation is to create a
# "unified" C++ source file which simply #include's other source files.  Rather
# than compiling the original source files, we compile only the unified file,
# which greatly reduces compilation overhead in terms of I/O, duplicate template
# instantiations, and the like. This makes compiling the project much faster.
#
# Each automake file (i.e., file with '.am' extension) in the source tree is
# examined to find any variables of the form 'foo_UNIFIED' and 'foo_NONUNIFIED'.
# The script generates makefile rules that create a '.cpp' file that #include's
# the 'foo_UNIFIED' files. The '.cpp' file is added to 'foo_SOURCES' so that it
# will be compiled, and the usual automake features like dependency tracking
# work as expected. 'foo_NONUNIFIED' files are added directly to 'foo_SOURCES'
# and continue to be compiled separately.
#
# Putting every 'foo_UNIFIED' file in the same generated '.cpp' file would
# maximize single-threaded compilation speed, but it would limit parallelism and
# slow down incremental builds. This is why the 'foo_UNIFIED' files are split
# into chunks, with each chunk getting its own '.cpp' file which is compiled
# separately. The size of the chunks is a tunable parameter; if the default
# value isn't working well, it can be overridden by providing a custom chunk
# size as a parameter to this script.
#
# Correct usage requires setting up the makefiles appropriately. Since this
# script manages 'foo_SOURCES' itself, specifying 'foo_SOURCES' is an error if
# 'foo_UNIFIED' or 'foo_NONUNIFIED' is specified. This helps avoid unexpected
# interactions. 'foo_UNIFIED' also can't include any expansions ("$FOO" or
# "$(foo)") or any other makefile trickery, and it can't include file types
# other than '.c' and '.cpp' files. That includes headers; if you really want to
# include them somewhere, 'foo_NONUNIFIED' is the place, but the best thing is
# to leave them out altogether and let automake's automatic dependency
# management handle them.
#
# It's not necessary to worry about excluding automake files that aren't
# relevant to the target you're building. This script only generates make rules;
# if you're not building a target that triggers them, they won't have any
# effect.
#
# The output of the script is written to stdout, so all that's necessary to use
# it is to run it from the root of the source tree before automake is run,
# redirect the output to a file, and then include that file from your
# Makefile.am. The script requires that Makefile.am also set a GEN_UNIFIED_CPP
# variable which points to the script that will actually create the generated
# '.cpp' files at build time. (That's gen-unified-sources.py.) Doing it this way
# allows this tool to run before the build directory has been created, and
# avoids generating '.cpp' files that aren't needed by the targets we're
# building.

from __future__ import print_function
import collections
import fnmatch
import math
import os
import re
import stat
import sys

def find_automake_files():
    files = []
    for root, _, filenames in os.walk('.', followlinks=True):
        for filename in fnmatch.filter(filenames, '*.am'):
            files.append(os.path.join(root, filename))
    return files

def construct_file_set(makefile, reldir, target, type, files):
    for file in files:
        # Filter empty filenames. These end up getting passed to us because of
        # split()'s behavior when the regular expression contains a capture
        # group.
        if file == '':
            continue

        # Replace %reldir% replaced with a concrete path.
        file = file.replace('%reldir%', reldir)
{'fraction_non_alphanumeric': 0.06887417218543046, 'fraction_numerical': 0.0017660044150110375, 'mean_word_length': 4.151222285389426, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2464170', 'n_tokens_mistral': 2443, 'n_tokens_neox': 2339, 'n_words': 1265}