<commit_before>#!/usr/bin/python
import sys
import argparse
import os

parser = argparse.ArgumentParser(description="Make helper parser")
parser.add_argument("--directory", dest="directory", required=True)
parser.add_argument("--library", dest="library", help="Make a library")
parser.add_argument("--binary", dest="binary", help="Make a binary")
parser.add_argument("--exclude", dest="exclude", help="Exclude file", action="append")
parser.add_argument("--include", dest="include", help="Include file", action="append")
parser.add_argument("--out", dest="out", help="Output file")
parser.add_argument("--cpp-suffix", dest="cpp_suffix", help="C++ file suffix")
PREFIX=None
LIBRARY=None
BINARY=None
EXCLUDE=[]
INCLUDE=[]
OUTFILE="targets.mk"
CPP_SUFFIX=".cpp"

def make_o(x):
    return os.path.splitext(x)[0] + ".o"

def write_cpp_rule_pattern(f):
    src = "$(%s_SRCDIR)/%%%s"%(PREFIX, CPP_SUFFIX)
    dst = "$(%s_SRCDIR)/%%.o"%(PREFIX)

    f.write("%s: %s\n"%(dst, src))
    f.write('\t$(QUIET_CXX)$(CXX) $(CFLAGS) $(CXXFLAGS) $(INCLUDES) $(' + PREFIX + '_CFLAGS) $(' + PREFIX + '_INCLUDES) -c $(CXX_O) $<\n')
    f.write("\n")

def write_c_rule_pattern(f):
    src = "$(%s_SRCDIR)/%%.c"%(PREFIX)
    dst = "$(%s_SRCDIR)/%%.o"%(PREFIX)

    f.write("%s: %s\n"%(dst, src))
    f.write('\t$(QUIET_CC)$(CC) $(CFLAGS) $(INCLUDES) $(' + PREFIX + '_CFLAGS) $(' + PREFIX + '_INCLUDES) -c $(CXX_O) $<\n')
    f.write("\n")

def write_asm_rule_pattern(f):
    src = "$(%s_SRCDIR)/%%.asm"%(PREFIX)
    dst = "$(%s_SRCDIR)/%%.o"%(PREFIX)

    f.write("%s: %s\n"%(dst, src))
    f.write('\t$(QUIET_ASM)$(ASM) $(ASMFLAGS) $(ASM_INCLUDES) $(' + PREFIX + '_ASMFLAGS) $(' + PREFIX + '_ASM_INCLUDES) -o $@ $<\n')
    f.write("\n")

def write_asm_s_rule_pattern(f):
    src = "$(%s_SRCDIR)/%%.S"%(PREFIX)
    dst = "$(%s_SRCDIR)/%%.o"%(PREFIX)

    f.write("%s: %s\n"%(dst, src))
    f.write('\t$(QUIET_CCAS)$(CCAS) $(CFLAGS) $(ASMFLAGS) $(INCLUDES) $(' + PREFIX + '_CFLAGS) $(' + PREFIX + '_INCLUDES) -c -o $@ $<\n')
    f.write("\n")


def find_sources():
    cpp_files = []
    asm_files = []
    c_files = []
    s_files = []
    print EXCLUDE
    for dir in os.walk("."):
        for file in dir[2]:
            if (len(INCLUDE) == 0 and not file in EXCLUDE) or file in INCLUDE:
                if os.path.splitext(file)[1] == CPP_SUFFIX:
                    cpp_files.append(os.path.join(dir[0].strip('./'), file))
                if os.path.splitext(file)[1] == '.asm':
                    asm_files.append(os.path.join(dir[0].strip('./'), file))
                if os.path.splitext(file)[1] == '.c':
                    c_files.append(os.path.join(dir[0].strip('./'), file))
                if os.path.splitext(file)[1] == '.S':
                    s_files.append(os.path.join(dir[0].strip('./'), file))
    return [cpp_files, asm_files, c_files, s_files]


args = parser.parse_args()

if args.library is not None:
    PREFIX=args.library.upper()
elif args.binary is not None:
    PREFIX=args.binary.upper()
else:
    sys.stderr.write("Must provide either library or binary")
    sys.exit(1)

if args.exclude is not None:
    EXCLUDE = args.exclude
if args.include is not None:
    INCLUDE = args.include
if args.out is not None:
    OUTFILE = args.out
else:
    OUTFILE = os.path.join(args.directory, OUTFILE)
if args.cpp_suffix is not None:
    CPP_SUFFIX = args.cpp_suffix

OUTFILE = os.path.abspath(OUTFILE)
try:
    os.chdir(args.directory)
except:
    sys.exit(1)

(cpp, asm, cfiles, sfiles) = find_sources()



f = open(OUTFILE, "w")
f.write("%s_SRCDIR=%s\n"%(PREFIX, args.directory))

f.write("%s_CPP_SRCS=\\\n"%(PREFIX))
for c in cpp:
    f.write("\t$(%s_SRCDIR)/%s\\\n"%(PREFIX, c))
f.write("\n")
f.write("%s_OBJS += $(%s_CPP_SRCS:%s=.o)\n\n"%(PREFIX, PREFIX, CPP_SUFFIX))

if len(cfiles) > 0:
    f.write("%s_C_SRCS=\\\n"%(PREFIX))
    for cfile in cfiles:
        f.write("\t$(%s_SRCDIR)/%s\\\n"%(PREFIX, cfile))
    f.write("\n")
    f.write("%s_OBJS += $(%s_C_SRCS:.c=.o)\n\n"%(PREFIX, PREFIX))
<commit_msg>Explicitly sort the files in mktargets.py<commit_after>#!/usr/bin/python
import sys
import argparse
import os

parser = argparse.ArgumentParser(description="Make helper parser")
parser.add_argument("--directory", dest="directory", required=True)
parser.add_argument("--library", dest="library", help="Make a library")
parser.add_argument("--binary", dest="binary", help="Make a binary")
parser.add_argument("--exclude", dest="exclude", help="Exclude file", action="append")
parser.add_argument("--include", dest="include", help="Include file", action="append")
parser.add_argument("--out", dest="out", help="Output file")
parser.add_argument("--cpp-suffix", dest="cpp_suffix", help="C++ file suffix")
PREFIX=None
LIBRARY=None
BINARY=None
EXCLUDE=[]
INCLUDE=[]
OUTFILE="targets.mk"
CPP_SUFFIX=".cpp"

def make_o(x):
    return os.path.splitext(x)[0] + ".o"

def write_cpp_rule_pattern(f):
    src = "$(%s_SRCDIR)/%%%s"%(PREFIX, CPP_SUFFIX)
    dst = "$(%s_SRCDIR)/%%.o"%(PREFIX)

    f.write("%s: %s\n"%(dst, src))
    f.write('\t$(QUIET_CXX)$(CXX) $(CFLAGS) $(CXXFLAGS) $(INCLUDES) $(' + PREFIX + '_CFLAGS) $(' + PREFIX + '_INCLUDES) -c $(CXX_O) $<\n')
    f.write("\n")

def write_c_rule_pattern(f):
    src = "$(%s_SRCDIR)/%%.c"%(PREFIX)
    dst = "$(%s_SRCDIR)/%%.o"%(PREFIX)

    f.write("%s: %s\n"%(dst, src))
    f.write('\t$(QUIET_CC)$(CC) $(CFLAGS) $(INCLUDES) $(' + PREFIX + '_CFLAGS) $(' + PREFIX + '_INCLUDES) -c $(CXX_O) $<\n')
    f.write("\n")

def write_asm_rule_pattern(f):
    src = "$(%s_SRCDIR)/%%.asm"%(PREFIX)
    dst = "$(%s_SRCDIR)/%%.o"%(PREFIX)

    f.write("%s: %s\n"%(dst, src))
    f.write('\t$(QUIET_ASM)$(ASM) $(ASMFLAGS) $(ASM_INCLUDES) $(' + PREFIX + '_ASMFLAGS) $(' + PREFIX + '_ASM_INCLUDES) -o $@ $<\n')
    f.write("\n")

def write_asm_s_rule_pattern(f):
    src = "$(%s_SRCDIR)/%%.S"%(PREFIX)
    dst = "$(%s_SRCDIR)/%%.o"%(PREFIX)

    f.write("%s: %s\n"%(dst, src))
    f.write('\t$(QUIET_CCAS)$(CCAS) $(CFLAGS) $(ASMFLAGS) $(INCLUDES) $(' + PREFIX + '_CFLAGS) $(' + PREFIX + '_INCLUDES) -c -o $@ $<\n')
    f.write("\n")


def find_sources():
    cpp_files = []
    asm_files = []
    c_files = []
    s_files = []
    print EXCLUDE
    for dir in os.walk("."):
        for file in dir[2]:
            if (len(INCLUDE) == 0 and not file in EXCLUDE) or file in INCLUDE:
                if os.path.splitext(file)[1] == CPP_SUFFIX:
                    cpp_files.append(os.path.join(dir[0].strip('./'), file))
                if os.path.splitext(file)[1] == '.asm':
                    asm_files.append(os.path.join(dir[0].strip('./'), file))
                if os.path.splitext(file)[1] == '.c':
                    c_files.append(os.path.join(dir[0].strip('./'), file))
                if os.path.splitext(file)[1] == '.S':
                    s_files.append(os.path.join(dir[0].strip('./'), file))
    return [cpp_files, asm_files, c_files, s_files]


args = parser.parse_args()

if args.library is not None:
    PREFIX=args.library.upper()
elif args.binary is not None:
    PREFIX=args.binary.upper()
else:
    sys.stderr.write("Must provide either library or binary")
    sys.exit(1)

if args.exclude is not None:
    EXCLUDE = args.exclude
if args.include is not None:
    INCLUDE = args.include
if args.out is not None:
    OUTFILE = args.out
else:
    OUTFILE = os.path.join(args.directory, OUTFILE)
if args.cpp_suffix is not None:
    CPP_SUFFIX = args.cpp_suffix

OUTFILE = os.path.abspath(OUTFILE)
try:
    os.chdir(args.directory)
except:
    sys.exit(1)

(cpp, asm, cfiles, sfiles) = find_sources()

cpp = sorted(cpp, key=lambda s: s.lower())
asm = sorted(asm, key=lambda s: s.lower())
cfiles = sorted(cfiles, key=lambda s: s.lower())
sfiles = sorted(sfiles, key=lambda s: s.lower())



f = open(OUTFILE, "w")
f.write("%s_SRCDIR=%s\n"%(PREFIX, args.directory))

f.write("%s_CPP_SRCS=\\\n"%(PREFIX))
for c in cpp:
    f.write("\t$(%s_SRCDIR)/%s\\\n"%(PREFIX, c))
f.write("\n")
f.write("%s_OBJS += $(%s_CPP_SRCS:%s=.o)\n\n"%(PREFIX, PREFIX, CPP_SUFFIX))

if len(cfiles) > 0:
    f.write("%s_C_SRCS=\\\n"%(PREFIX))
    for cfile in cfiles:
        f.write("\t$(%s_SRCDIR)/%s\\\n"%(PREFIX, cfile))
    f.write("\n")
    f.write("%s_OBJS += $(%s_C_SRCS:.c=.o)\n\n"%(PREFIX, PREFIX))

{'fraction_non_alphanumeric': 0.22096832908627595, 'fraction_numerical': 0.003397645916757675, 'mean_word_length': 3.508752735229759, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7105035', 'n_tokens_mistral': 3362, 'n_tokens_neox': 3197, 'n_words': 597}