<commit_before>############################################################################ 
# 
#  Copyright 2017 
# 
#   https://github.com/shahzebsiddiqui/BuildTest 
# 
#  This file is part of buildtest. 
# 
#    buildtest is free software: you can redistribute it and/or modify 
#    it under the terms of the GNU General Public License as published by 
#    the Free Software Foundation, either version 3 of the License, or 
#    (at your option) any later version. 
# 
#    buildtest is distributed in the hope that it will be useful, 
#    but WITHOUT ANY WARRANTY; without even the implied warranty of 
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
#    GNU General Public License for more details. 
# 
#    You should have received a copy of the GNU General Public License 
#    along with buildtest.  If not, see <http://www.gnu.org/licenses/>. 
############################################################################# 

from setup import *
from utilities import *
import os.path 
import os, sys
import shutil
import yaml

def systempkg_generate_binary_test(pkg,verbose,logdir):
	"""
	This function generates the binary test for system packages by processing 
	command.yaml. We make sure command.yaml exists and CMakeLists.txt is present
	in each subdirectory in BUILDTEST_TESTDIR. If CMakeLists.txt is not present, we 
	create the file and add the content "add_subdirectory(<dirname>) according to
	which directory you want ctest to process
 	
	"""
	toplevel_cmakelist_file=os.path.join(BUILDTEST_ROOT,"CMakeLists.txt")
	testingdir_cmakelist_file=os.path.join(BUILDTEST_TESTDIR,"CMakeLists.txt")
	commandfile=os.path.join(BUILDTEST_SOURCEDIR,"system",pkg,"command.yaml")

        logcontent = ""
        logcontent += "---------------------------------------------- \n "
        logcontent += "function: systempkg_generate_binary_test \n"
        logcontent += "---------------------------------------------- \n "

        # if CMakeLists.txt does not exist in top-level directory, create the header
        if os.path.isfile(toplevel_cmakelist_file) == False:
                logcontent += "File: " + toplevel_cmakelist_file + " was not found, build test will create it \n"
                init_CMakeList(toplevel_cmakelist_file)

        logcontent += "Creating Directory " + BUILDTEST_TESTDIR + "\n"
        create_dir(BUILDTEST_TESTDIR,verbose)

        # if BUILDTEST_TESTDIR/CMakeLists.txt does not exist, then create it
        if os.path.isfile(testingdir_cmakelist_file) == False:
	        logcontent += "File: " + testingdir_cmakelist_file + " was not found, build test will create it \n"
                fd=open(testingdir_cmakelist_file,'w')
                fd.close()
        # if command.yaml does not exist then report warning and skip to next test.
        if os.path.isfile(commandfile) == False:
                print "Warning: Cannot find command file:", commandfile, "Skipping binary test"
		logcontent += "Warning: Cannot find command file:" + commandfile + "Skipping binary test"
        else:
            logcontent += systempkg_process_binary_file(commandfile,pkg,verbose,logdir)

	return logcontent
def generate_binary_test(software,toolchain,configdir,verbose,logdir):
	"""
	This function operates similar to systempkg_generate_binary_test except this function
	is used only for ebapps that are present as modules. This is because ebapps names are 
	different then systempkg names for instance "gcc" "gcc-gfortran" "gcc-c++" are package 
	names in RHEL that test binaries for gcc, gfortran, g++. In EB there is a single app GCC
	that provides gcc, gfortran, g++. This function makes sure command.yaml exists and CMakeLists.txt
	is present in all subdirectories in BUILDTEST_TESTDIR
	"""
	toplevel_cmakelist_file=os.path.join(BUILDTEST_ROOT,"CMakeLists.txt")
	testingdir_cmakelist_file=os.path.join(BUILDTEST_TESTDIR,"CMakeLists.txt")
	swname = software[0]
	commandfile=os.path.join(configdir,"command.yaml")

	logcontent = ""
	logcontent += "--------------------------------- \n "
	logcontent += "function: generate_binary_test \n"
	logcontent += "--------------------------------- \n "
	# if CMakeLists.txt does not exist in top-level directory, create the header
	if os.path.isfile(toplevel_cmakelist_file) == False:
		logcontent += "File: " + toplevel_cmakelist_file + " was not found, build test will create it \n"
		init_CMakeList(toplevel_cmakelist_file)

	logcontent += "Creating Directory " + BUILDTEST_TESTDIR + "\n"
	create_dir(BUILDTEST_TESTDIR,verbose)

	# if BUILDTEST_TESTDIR/CMakeLists.txt does not exist, then create it
	if os.path.isfile(testingdir_cmakelist_file) == False:
		logcontent += "File: " + testingdir_cmakelist_file + " was not found, build test will create it \n"
		fd=open(testingdir_cmakelist_file,'w')
		fd.close()
	# if command.yaml does not exist then report error
	if os.path.isfile(commandfile) == False:
		print "Warning: Cannot find command file:", commandfile, "Skipping binary test"
		logcontent += "Warning: Cannot find command file:" + commandfile + "Skipping binary test"
	else:
	    logcontent += process_binary_file(commandfile,software,toolchain,verbose,logdir)

	return logcontent
# generate test for source
def generate_source_test(software,toolchain,configmap,codedir,verbose,subdir,logdir):
	"""
	This function generates the tests that requires compilation for EB apps. The
	tests are written <software>/<version>/<toolchain-name>/<toolchain-version>.
	The test script is named according to "name" key tag with the extension ".sh"
	CMakeLists.txt has an entry for each test that executes the shell-script. Most
	test requires a compilation step, while every test requires a execution stage. 
	This is done via buildcmd and runcmd tags in YAML for explicit builds. buildtest
	will try to generate this automatically if nothing is provided. 
	"""
	
	appname=software[0]
	appver=software[1]
	tcname=toolchain[0]
	tcver=toolchain[1]
	
	# app_destdir is root of test directory
	app_destdir = os.path.join(BUILDTEST_TESTDIR,"ebapp",appname,appver,tcname,tcver)

	# destdir is where test script and CMakeLists.txt will be generated. 
	# If there is a subdirectory then testscript will reside in subdir
	destdir=os.path.join(app_destdir,subdir)
	cmakelist=os.path.join(destdir,"CMakeLists.txt")
	
	logcontent = ""
	logcontent += "------------------------------------------------ \n"
	logcontent += "function: generate_source_test \n"
	logcontent += "------------------------------------------------ \n"

	# if subdirectory exists, create subdirectory in destdir so we can write test script
	if subdir != "":
		os.mkdir(destdir)

	# testname is key value "name" with .sh extension
	testname=configmap["name"]+".sh"
	testpath=os.path.join(destdir,testname)
	sourcefilepath=os.path.join(codedir,configmap["source"])
	# name of the executable is the value of source tag with ".exe" extension
	executable=configmap["source"]+".exe"

	flags=""
	# if buildopts key exists in dictionary, then add flags to compilation step (buildcmd)
	if "buildopts" in configmap:
		flags=configmap["buildopts"]

        logcontent += "Test Name: " +  testname + "\n"
	logcontent += "Test Path: " + testpath + "\n"
        logcontent += "Source File: " + sourcefilepath + "\n"
        logcontent += "Executable Name: " +  executable + "\n"<commit_msg>use os.makedirs instead of os.mkdir when YAML files were in subdirectories in the case of R and Python. When multiple examples are in subdirectories os.mkdir failed because directory exists already, so added a check before creating directory<commit_after>############################################################################ 
# 
#  Copyright 2017 
# 
#   https://github.com/shahzebsiddiqui/BuildTest 
# 
#  This file is part of buildtest. 
# 
#    buildtest is free software: you can redistribute it and/or modify 
#    it under the terms of the GNU General Public License as published by 
#    the Free Software Foundation, either version 3 of the License, or 
#    (at your option) any later version. 
# 
#    buildtest is distributed in the hope that it will be useful, 
#    but WITHOUT ANY WARRANTY; without even the implied warranty of 
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
#    GNU General Public License for more details. 
# 
#    You should have received a copy of the GNU General Public License 
#    along with buildtest.  If not, see <http://www.gnu.org/licenses/>. 
############################################################################# 

from setup import *
from utilities import *
import os.path 
import os, sys
import shutil
import yaml

def systempkg_generate_binary_test(pkg,verbose,logdir):
	"""
	This function generates the binary test for system packages by processing 
	command.yaml. We make sure command.yaml exists and CMakeLists.txt is present
	in each subdirectory in BUILDTEST_TESTDIR. If CMakeLists.txt is not present, we 
	create the file and add the content "add_subdirectory(<dirname>) according to
	which directory you want ctest to process
 	
	"""
	toplevel_cmakelist_file=os.path.join(BUILDTEST_ROOT,"CMakeLists.txt")
	testingdir_cmakelist_file=os.path.join(BUILDTEST_TESTDIR,"CMakeLists.txt")
	commandfile=os.path.join(BUILDTEST_SOURCEDIR,"system",pkg,"command.yaml")

        logcontent = ""
        logcontent += "---------------------------------------------- \n "
        logcontent += "function: systempkg_generate_binary_test \n"
        logcontent += "---------------------------------------------- \n "

        # if CMakeLists.txt does not exist in top-level directory, create the header
        if os.path.isfile(toplevel_cmakelist_file) == False:
                logcontent += "File: " + toplevel_cmakelist_file + " was not found, build test will create it \n"
                init_CMakeList(toplevel_cmakelist_file)

        logcontent += "Creating Directory " + BUILDTEST_TESTDIR + "\n"
        create_dir(BUILDTEST_TESTDIR,verbose)

        # if BUILDTEST_TESTDIR/CMakeLists.txt does not exist, then create it
        if os.path.isfile(testingdir_cmakelist_file) == False:
	        logcontent += "File: " + testingdir_cmakelist_file + " was not found, build test will create it \n"
                fd=open(testingdir_cmakelist_file,'w')
                fd.close()
        # if command.yaml does not exist then report warning and skip to next test.
        if os.path.isfile(commandfile) == False:
                print "Warning: Cannot find command file:", commandfile, "Skipping binary test"
		logcontent += "Warning: Cannot find command file:" + commandfile + "Skipping binary test"
        else:
            logcontent += systempkg_process_binary_file(commandfile,pkg,verbose,logdir)

	return logcontent
def generate_binary_test(software,toolchain,configdir,verbose,logdir):
	"""
	This function operates similar to systempkg_generate_binary_test except this function
	is used only for ebapps that are present as modules. This is because ebapps names are 
	different then systempkg names for instance "gcc" "gcc-gfortran" "gcc-c++" are package 
	names in RHEL that test binaries for gcc, gfortran, g++. In EB there is a single app GCC
	that provides gcc, gfortran, g++. This function makes sure command.yaml exists and CMakeLists.txt
	is present in all subdirectories in BUILDTEST_TESTDIR
	"""
	toplevel_cmakelist_file=os.path.join(BUILDTEST_ROOT,"CMakeLists.txt")
	testingdir_cmakelist_file=os.path.join(BUILDTEST_TESTDIR,"CMakeLists.txt")
	swname = software[0]
	commandfile=os.path.join(configdir,"command.yaml")

	logcontent = ""
	logcontent += "--------------------------------- \n "
	logcontent += "function: generate_binary_test \n"
	logcontent += "--------------------------------- \n "
	# if CMakeLists.txt does not exist in top-level directory, create the header
	if os.path.isfile(toplevel_cmakelist_file) == False:
		logcontent += "File: " + toplevel_cmakelist_file + " was not found, build test will create it \n"
		init_CMakeList(toplevel_cmakelist_file)

	logcontent += "Creating Directory " + BUILDTEST_TESTDIR + "\n"
	create_dir(BUILDTEST_TESTDIR,verbose)

	# if BUILDTEST_TESTDIR/CMakeLists.txt does not exist, then create it
	if os.path.isfile(testingdir_cmakelist_file) == False:
		logcontent += "File: " + testingdir_cmakelist_file + " was not found, build test will create it \n"
		fd=open(testingdir_cmakelist_file,'w')
		fd.close()
	# if command.yaml does not exist then report error
	if os.path.isfile(commandfile) == False:
		print "Warning: Cannot find command file:", commandfile, "Skipping binary test"
		logcontent += "Warning: Cannot find command file:" + commandfile + "Skipping binary test"
	else:
	    logcontent += process_binary_file(commandfile,software,toolchain,verbose,logdir)

	return logcontent
# generate test for source
def generate_source_test(software,toolchain,configmap,codedir,verbose,subdir,logdir):
	"""
	This function generates the tests that requires compilation for EB apps. The
	tests are written <software>/<version>/<toolchain-name>/<toolchain-version>.
	The test script is named according to "name" key tag with the extension ".sh"
	CMakeLists.txt has an entry for each test that executes the shell-script. Most
	test requires a compilation step, while every test requires a execution stage. 
	This is done via buildcmd and runcmd tags in YAML for explicit builds. buildtest
	will try to generate this automatically if nothing is provided. 
	"""
	
	appname=software[0]
	appver=software[1]
	tcname=toolchain[0]
	tcver=toolchain[1]
	
	# app_destdir is root of test directory
	app_destdir = os.path.join(BUILDTEST_TESTDIR,"ebapp",appname,appver,tcname,tcver)

	# destdir is where test script and CMakeLists.txt will be generated. 
	# If there is a subdirectory then testscript will reside in subdir
	destdir=os.path.join(app_destdir,subdir)
	cmakelist=os.path.join(destdir,"CMakeLists.txt")
	
	logcontent = ""
	logcontent += "------------------------------------------------ \n"
	logcontent += "function: generate_source_test \n"
	logcontent += "------------------------------------------------ \n"

	# if subdirectory exists, create subdirectory in destdir so we can write test script
	if subdir != "":
		# if sub directory does not exist, then create all directories and its parents directories
		if os.path.exists(destdir) == False:
			os.makedirs(destdir)

	# testname is key value "name" with .sh extension
	testname=configmap["name"]+".sh"
	testpath=os.path.join(destdir,testname)
	sourcefilepath=os.path.join(codedir,configmap["source"])
	# name of the executable is the value of source tag with ".exe" extension
	executable=configmap["source"]+".exe"

	flags=""
	# if buildopts key exists in dictionary, then add flags to compilation step (buildcmd)
	if "buildopts" in configmap:
		flags=configmap["buildopts"]

        logcontent += "Test Name: " +  testname + "\n"
	logcontent += "Test Path: " + testpath + "\n"
        logcontent += "Source File: " + sourcefilepath + "\n"
        logcontent += "Executable Name: " +  executable + "\n"
{'fraction_non_alphanumeric': 0.1393398988554698, 'fraction_numerical': 0.0013308490817141336, 'mean_word_length': 3.5974304068522485, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1614564', 'n_tokens_mistral': 4536, 'n_tokens_neox': 4288, 'n_words': 1410}