<commit_before>from __future__ import print_function
import re
import argparse
import os.path

parser = argparse.ArgumentParser()
parser.add_argument('version', help='version to include')
parser.add_argument('outfile', help='outfile w/o extension')
parser.add_argument('inputs', nargs='*', action='store', help='input filenames')
args = parser.parse_args()

outname = args.outfile.split("/")[-1]
pos = outname.find(".")
if pos > 0:
    outname = outname[:pos]
include_re = re.compile("^#include ([\"<].*[\">]).*$")
guard_re = re.compile("^#(?:(?:ifndef|define) [A-Z_]+_H_|endif /\* [A-Z_]+_H_ \*/)")
includes = []

is_c = False

print ("Starting amalgamating file "+ args.outfile)

for fname in args.inputs:
    if("util.h" in fname):
        is_c = True
        continue
    with open(fname, encoding="utf8") as infile:
        print ("Integrating file '" + fname + "'...", end=""),
        for line in infile:
            res = include_re.match(line)
            if res:
                inc = res.group(1)
                if not inc in includes and not inc[0] == '"':
                    includes.append(inc)
        print ("done."),

file = open(args.outfile, 'w')
file.write('''/* THIS IS A SINGLE-FILE DISTRIBUTION CONCATENATED FROM THE OPEN62541 SOURCES 
 * visit http://open62541.org/ for information about this software
 * Git-Revision: %s
 */
 
 /*
 * Copyright (C) 2015 the contributors as stated in the AUTHORS file
 *
 * This file is part of open62541. open62541 is free software: you can
 * redistribute it and/or modify it under the terms of the GNU Lesser General
 * Public License, version 3 (as published by the Free Software Foundation) with
 * a static linking exception as stated in the LICENSE file provided with
 * open62541.
 *
 * open62541 is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
 * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
 * details.
 */\n\n''' % args.version)

if not is_c:
    file.write('''#ifndef %s
#define %s

#ifdef __cplusplus
extern "C" {
#endif\n\n''' % (outname.upper() + "_H_", outname.upper() + "_H_") )

if not is_c:
    for inc in includes:
        file.write("#include " + inc + "\n")
else:
    file.write("#define UA_AMALGAMATE\n")
    file.write('''#ifndef UA_DYNAMIC_LINKING
# define UA_DYNAMIC_LINKING
#endif\n\n''')
    for fname in args.inputs:
        if "ua_config.h" in fname or "ua_util.h" in fname:
            with open(fname, encoding="utf8") as infile:
                print ("Integrating file '" + fname + "'...", end=""),
                for line in infile:
                    file.write(line)
                print ("done."),
    file.write("#include \"" + outname + ".h\"\n")

for fname in args.inputs:
    if not "util.h" in fname:
        with open(fname, encoding="utf8") as infile:
            file.write("/*********************************** amalgamated original file \"" + fname + "\" ***********************************/\n")
            print ("Integrating file '" + fname + "'...", end=""),
            for line in infile:
                inc_res = include_re.match(line)
                guard_res = guard_re.match(line)
                if not inc_res and not guard_res:
                    file.write(line)
            print ("done."),

if not is_c:
    file.write('''
#ifdef __cplusplus
} // extern "C"
#endif

#endif /* %s */''' % (outname.upper() + "_H_"))
file.close()

print ("The size of "+args.outfile+" is "+ str(os.path.getsize(args.outfile))+" Bytes.")
<commit_msg>using io.open instead of open to have the encoding parameter available on python 2.7, too.<commit_after>from __future__ import print_function
import re
import argparse
import os.path
import io

parser = argparse.ArgumentParser()
parser.add_argument('version', help='version to include')
parser.add_argument('outfile', help='outfile w/o extension')
parser.add_argument('inputs', nargs='*', action='store', help='input filenames')
args = parser.parse_args()

outname = args.outfile.split("/")[-1]
pos = outname.find(".")
if pos > 0:
    outname = outname[:pos]
include_re = re.compile("^#include ([\"<].*[\">]).*$")
guard_re = re.compile("^#(?:(?:ifndef|define) [A-Z_]+_H_|endif /\* [A-Z_]+_H_ \*/)")
includes = []

is_c = False

print ("Starting amalgamating file "+ args.outfile)

for fname in args.inputs:
    if("util.h" in fname):
        is_c = True
        continue
    with io.open(fname, encoding="utf8") as infile:
        print ("Integrating file '" + fname + "'...", end=""),
        for line in infile:
            res = include_re.match(line)
            if res:
                inc = res.group(1)
                if not inc in includes and not inc[0] == '"':
                    includes.append(inc)
        print ("done."),

file = io.open(args.outfile, 'w')
file.write('''/* THIS IS A SINGLE-FILE DISTRIBUTION CONCATENATED FROM THE OPEN62541 SOURCES 
 * visit http://open62541.org/ for information about this software
 * Git-Revision: %s
 */
 
 /*
 * Copyright (C) 2015 the contributors as stated in the AUTHORS file
 *
 * This file is part of open62541. open62541 is free software: you can
 * redistribute it and/or modify it under the terms of the GNU Lesser General
 * Public License, version 3 (as published by the Free Software Foundation) with
 * a static linking exception as stated in the LICENSE file provided with
 * open62541.
 *
 * open62541 is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
 * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
 * details.
 */\n\n''' % args.version)

if not is_c:
    file.write('''#ifndef %s
#define %s

#ifdef __cplusplus
extern "C" {
#endif\n\n''' % (outname.upper() + "_H_", outname.upper() + "_H_") )

if not is_c:
    for inc in includes:
        file.write("#include " + inc + "\n")
else:
    file.write("#define UA_AMALGAMATE\n")
    file.write('''#ifndef UA_DYNAMIC_LINKING
# define UA_DYNAMIC_LINKING
#endif\n\n''')
    for fname in args.inputs:
        if "ua_config.h" in fname or "ua_util.h" in fname:
            with io.open(fname, encoding="utf8") as infile:
                print ("Integrating file '" + fname + "'...", end=""),
                for line in infile:
                    file.write(line)
                print ("done."),
    file.write("#include \"" + outname + ".h\"\n")

for fname in args.inputs:
    if not "util.h" in fname:
        with io.open(fname, encoding="utf8") as infile:
            file.write("/*********************************** amalgamated original file \"" + fname + "\" ***********************************/\n")
            print ("Integrating file '" + fname + "'...", end=""),
            for line in infile:
                inc_res = include_re.match(line)
                guard_res = guard_re.match(line)
                if not inc_res and not guard_res:
                    file.write(line)
            print ("done."),

if not is_c:
    file.write('''
#ifdef __cplusplus
} // extern "C"
#endif

#endif /* %s */''' % (outname.upper() + "_H_"))
file.close()

print ("The size of "+args.outfile+" is "+ str(os.path.getsize(args.outfile))+" Bytes.")

{'fraction_non_alphanumeric': 0.16315861879213098, 'fraction_numerical': 0.01183106342000275, 'mean_word_length': 2.81827731092437, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6676709', 'n_tokens_mistral': 2368, 'n_tokens_neox': 2258, 'n_words': 706}