<commit_before>#! /usr/bin/env python
# coding:utf-8

###############################################################################
#
#                           The Cling Interpreter
#
# Cling Packaging Tool (CPT)
#
# tools/packaging/cpt.py: Main script which calls other helper scripts to
# compile and package Cling for multiple platforms.
#
# TODO: Add documentation here, or provide link to documentation
#
# Author: <NAME> <<EMAIL>>
#
# This file is dual-licensed: you can choose to license it under the University
# of Illinois Open Source License or the GNU Lesser General Public License. See
# LICENSE.TXT for details.
#
###############################################################################

import argparse
import urllib2
import os
import sys
import platform
import subprocess
import shutil
import glob
import re
import tarfile
from email.utils import formatdate
from datetime import tzinfo
import time
import multiprocessing
import fileinput

###############################################################################
#                           Platform initialization                           #
###############################################################################

OS=platform.system()
FAMILY=os.name.upper()

if OS == 'Windows':
    RELEASE = OS + ' ' + platform.release()
    REV = platform.version()

    EXEEXT = '.exe'
    SHLIBEXT = '.dll'

    TMP_PREFIX='C:/Windows/Temp/cling-obj/'
    workdir = 'C:/ec/build'

elif OS == 'Linux':
    DIST = platform.linux_distribution()[0]
    RELEASE = platform.linux_distribution()[2]
    REV = platform.linux_distribution()[1]

    EXEEXT = ''
    SHLIBEXT = '.so'

    TMP_PREFIX='/var/tmp/cling-obj/'
    workdir = os.path.expanduser('~/ec/build')

elif OS == 'Darwin':
    RELEASE = platform.release()
    REV = platform.mac_ver()[0]

    EXEEXT = ''
    SHLIBEXT = '.dylib'

    TMP_PREFIX='/var/tmp/cling-obj/'
    workdir = os.path.expanduser('~/ec/build')

else:
    # Extensions will be detected anyway by set_ext()
    EXEEXT = ''
    SHLIBEXT = ''

    #TODO: Need to test this in other platforms
    TMP_PREFIX='/var/tmp/cling-obj/'
    workdir = os.path.expanduser('~/ec/build')

if RELEASE == '':
    RELEASE = 'N/A'

if DIST == '':
    DIST = 'N/A'

if REV == '':
    REV = 'N/A'


###############################################################################
#                               Global variables                              #
###############################################################################

srcdir = os.path.join(workdir, 'cling-src')
CLING_SRC_DIR = os.path.join(srcdir, 'tools/cling')
LLVM_OBJ_ROOT = os.path.join(workdir, 'builddir')
prefix = ''
LLVM_GIT_URL = 'http://root.cern.ch/git/llvm.git'
CLANG_GIT_URL = 'http://root.cern.ch/git/clang.git'
CLING_GIT_URL = 'http://root.cern.ch/git/cling.git'
LLVMRevision = urllib2.urlopen("https://raw.githubusercontent.com/ani07nov/cling/master/LastKnownGoodLLVMSVNRevision.txt").readline().strip()
VERSION=''


###############################################################################
#              Platform independent functions (formerly indep.py)             #
###############################################################################

def exec_subprocess_call(cmd, cwd):
    if OS == 'Windows':
        subprocess.Popen(cmd.split(),
                         cwd=cwd,
                         shell=True,
                         stdin=subprocess.PIPE,
                         stdout=None,
                         stderr=subprocess.STDOUT).communicate()
    else:
        subprocess.Popen([cmd],
                         cwd=cwd,
                         shell=True,
                         stdin=subprocess.PIPE,
                         stdout=None,
                         stderr=subprocess.STDOUT).communicate()

def exec_subprocess_check_output(cmd, cwd):
    if OS == 'Windows':
        return subprocess.Popen(cmd.split(),
                     cwd=cwd,
                     shell=True,
                     stdin=subprocess.PIPE,
                     stdout=subprocess.PIPE,
                     stderr=subprocess.STDOUT).communicate()[0]
    else:
        return subprocess.Popen([cmd],
                     cwd=cwd,
                     shell=True,
                     stdin=subprocess.PIPE,
                     stdout=subprocess.PIPE,
                     stderr=subprocess.STDOUT).communicate()[0]


def box_draw_header():
    msg='cling (' + platform.machine() + ')' + formatdate(time.time(),tzinfo())
    spaces_no = 80 - len(msg) - 4
    spacer = ' ' * spaces_no
    msg='cling (' + platform.machine() + ')' + spacer + formatdate(time.time(),tzinfo())

    if OS != 'Windows':
        print '''
╔══════════════════════════════════════════════════════════════════════════════╗
║ %s ║
╚══════════════════════════════════════════════════════════════════════════════╝'''%(msg)
    else:
        print '''
+=============================================================================+
| %s |
+=============================================================================+'''%(msg)


def box_draw(msg):
    spaces_no = 80 - len(msg) - 4
    spacer = ' ' * spaces_no

    if OS != 'Windows':
        print '''
┌──────────────────────────────────────────────────────────────────────────────┐
│ %s%s │
└──────────────────────────────────────────────────────────────────────────────┘'''%(msg, spacer)
    else:
        print '''
+-----------------------------------------------------------------------------+
| %s%s |
+-----------------------------------------------------------------------------+'''%(msg, spacer)


def fetch_llvm():
    box_draw("Fetch source files")
    print "Last known good LLVM revision is: %s"%(LLVMRevision)
    def get_fresh_llvm():
        exec_subprocess_call('git clone %s %s'%(LLVM_GIT_URL, srcdir), workdir)

        exec_subprocess_call('git checkout ROOT-patches-r%s'%(LLVMRevision), srcdir)

    def update_old_llvm():
        exec_subprocess_call('git stash', srcdir)

        exec_subprocess_call('git clean -f -x -d', srcdir)

        exec_subprocess_call('git fetch --tags', srcdir)

        exec_subprocess_call('git checkout ROOT-patches-r%s'%(LLVMRevision), srcdir)

        exec_subprocess_call('git pull origin refs/tags/ROOT-patches-r%s'%(LLVMRevision), srcdir)

    if os.path.isdir(srcdir):
        update_old_llvm()
    else:
        get_fresh_llvm()


def fetch_clang():
    def get_fresh_clang():

        exec_subprocess_call('git clone %s %s/tools/clang'%(CLANG_GIT_URL, srcdir), os.path.join(srcdir, 'tools'))

        exec_subprocess_call('git checkout ROOT-patches-r%s'%(LLVMRevision), os.path.join(srcdir, 'tools', 'clang'))

    def update_old_clang():
        exec_subprocess_call('git stash', os.path.join(srcdir, 'tools', 'clang'))

        exec_subprocess_call('git clean -f -x -d', os.path.join(srcdir, 'tools', 'clang'))

        exec_subprocess_call('git fetch --tags', os.path.join(srcdir, 'tools', 'clang'))

        exec_subprocess_call('git checkout ROOT-patches-r%s'%(LLVMRevision), os.path.join(srcdir, 'tools', 'clang'))

        exec_subprocess_call('git pull origin refs/tags/ROOT-patches-r%s'%(LLVMRevision), os.path.join(srcdir, 'tools', 'clang'))

    if os.path.isdir(os.path.join(srcdir, 'tools' 'clang')):
        update_old_clang()
    else:
        get_fresh_clang()

def fetch_cling(arg):
    def get_fresh_cling():
        exec_subprocess_call('git clone %s %s'%(CLING_GIT_URL, CLING_SRC_DIR), os.path.join(srcdir, 'tools'))

        if arg == 'last-stable':
            checkout_branch = exec_subprocess_check_output('git describe --match v* --abbrev=0 --tags | head -n 1', CLING_SRC_DIR)

        elif arg == 'master':
            checkout_branch = 'master'
        else:
            checkout_branch = arg

        exec_subprocess_call('git checkout %s'%(checkout_branch), CLING_SRC_DIR)

    def update_old_cling():
        exec_subprocess_call('git stash', CLING_SRC_DIR)

        exec_subprocess_call('git clean -f -x -d', CLING_SRC_DIR)

        exec_subprocess_call('git fetch --tags', CLING_SRC_DIR)

        if arg == 'last-stable':
            checkout_branch = exec_subprocess_check_output('git describe --match v* --abbrev=0 --tags | head -n 1', CLING_SRC_DIR)

        elif arg == 'master':
            checkout_branch = 'master'
        else:
            checkout_branch = arg

        exec_subprocess_call('git checkout %s'%(checkout_branch), CLING_SRC_DIR)

        exec_subprocess_call('git pull origin %s'%(checkout_branch), CLING_SRC_DIR)

    if os.path.isdir(CLING_SRC_DIR):
        update_old_cling()
    else:
        get_fresh_cling()


def set_version():
    global VERSION
    box_draw("Set Cling version")
    VERSION=open(CLING_SRC_DIR+'/VERSION', 'r').readline().strip()

    # If development release, then add revision to the version
    REVISION = exec_subprocess_check_output('git log -n 1 --pretty=format:%H', CLING_SRC_DIR).strip()

    if '~dev' in VERSION:
        VERSION = VERSION + '-' + REVISION[:7]

    print 'Version: ' + VERSION
    print 'Revision: ' + REVISION


def set_ext():
    global EXEEXT
    global SHLIBEXT
    box_draw("Set binary/library extensions")
    if not os.path.isfile(LLVM_OBJ_ROOT + '/test/lit.site.cfg'):
        exec_subprocess_call('make lit.site.cfg', os.path.join(LLVM_OBJ_ROOT, 'test'))

    with open(LLVM_OBJ_ROOT + '/test/lit.site.cfg', 'r') as lit_site_cfg:
        for line in lit_site_cfg:
            if re.match('^config.llvm_shlib_ext = ', line):
                SHLIBEXT = re.sub('^config.llvm_shlib_ext = ', '', line).replace('"', '').strip()
            elif re.match('^config.llvm_exe_ext = ', line):
                EXEEXT = re.sub('^config.llvm_exe_ext = ', '', line).replace('"', '').strip()

    print 'EXEEXT: ' + EXEEXT
    print 'SHLIBEXT: ' + SHLIBEXT


def compile(arg):
    global prefix
    prefix=arg
    python=sys.executable
    cores=multiprocessing.cpu_count()

    # Cleanup previous installation directory if any
    if os.path.isdir(prefix):
        print "Remove directory: " + prefix
        shutil.rmtree(prefix)

    # Cleanup previous build directory if exists
    if os.path.isdir(workdir + '/builddir'):
        print "Remove directory: " + workdir + '/builddir'
        shutil.rmtree(workdir + '/builddir')

    os.makedirs(workdir + '/builddir')

    if platform.system() == 'Windows':
        box_draw("Configure Cling with CMake and generate Visual Studio 11 project files")
        exec_subprocess_call('cmake -G "Visual Studio 11" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=%s ../os.path.basename(%s)'%(TMP_PREFIX, srcdir), LLVM_OBJ_ROOT)

        box_draw("Building Cling (using %s cores)"%(cores))
        exec_subprocess_call('cmake --build . --target clang --config Release', LLVM_OBJ_ROOT)

        exec_subprocess_call('cmake --build . --target cling --config Release', LLVM_OBJ_ROOT)

        box_draw("Install compiled binaries to prefix (using %s cores)"%(cores))
        exec_subprocess_call('cmake --build . --target INSTALL --config Release', LLVM_OBJ_ROOT)

    else:
        box_draw("Configure Cling with GNU Make")
        exec_subprocess_call('%s/configure --disable-compiler-version-checks --with-python=%s --enable-targets=host --prefix=%s --enable-optimized=yes --enable-cxx11'%(srcdir, python, TMP_PREFIX), LLVM_OBJ_ROOT)

        box_draw("Building Cling (using %s cores)"%(cores))
        exec_subprocess_call('make -j%s'%(cores), LLVM_OBJ_ROOT)

        box_draw("Install compiled binaries to prefix (using %s cores)"%(cores))
        exec_subprocess_call('make install -j%s prefix=%s'%(cores, TMP_PREFIX), LLVM_OBJ_ROOT)


def install_prefix():
    set_ext()
    box_draw("Filtering Cling's libraries and binaries")
    print "This is going to take a while. Please wait."

    for line in fileinput.input(CLING_SRC_DIR + '/tools/packaging/dist-files.mk', inplace=True):
        if '@EXEEXT@' in line:
            print line.replace('@EXEEXT@', EXEEXT),
        elif '@SHLIBEXT@' in line:
            print line.replace('@SHLIBEXT@', SHLIBEXT),
        else:
            print line,

    dist_files = open(CLING_SRC_DIR + '/tools/packaging/dist-files.mk', 'r').read()
    for root, dirs, files in os.walk(TMP_PREFIX):
        for file in files:
            f=os.path.join(root, file).replace(TMP_PREFIX, '')
            if f+' ' in dist_files:
                print "Filter: " + f
                if not os.path.isdir(os.path.join(prefix,os.path.dirname(f))):
		    os.makedirs(os.path.join(prefix,os.path.dirname(f)))
                shutil.copy(os.path.join(TMP_PREFIX,f), os.path.join(prefix,f))

def test_cling():
    box_draw("Run Cling test suite")
    if platform.system() != 'Windows':
        exec_subprocess_call('make test', workdir + '/builddir/tools/cling')


def tarball():
    box_draw("Compress binaries into a bzip2 tarball")
    tar = tarfile.open(prefix+'.tar.bz2', 'w:bz2')
    tar.add(prefix, arcname=os.path.basename(prefix))
    tar.close()


def cleanup():
    print "\n"
    box_draw("Clean up")
    if os.path.isdir(os.path.join(workdir,'builddir')):
        print "Remove directory: " + os.path.join(workdir,'builddir')
        shutil.rmtree(os.path.join(workdir,'builddir'))

    if os.path.isdir(prefix):
        print "Remove directory: " + prefix
        shutil.rmtree(prefix)

    if os.path.isdir(TMP_PREFIX):
        print "Remove directory: " + TMP_PREFIX
        shutil.rmtree(TMP_PREFIX)

    if os.path.isfile(os.path.join(workdir,'cling.nsi')):
        print "Remove file: " + os.path.join(workdir,'cling.nsi')
        os.remove(os.path.join(workdir,'cling.nsi'))

    if args['current_dev'] == 'deb' or args['last_stable'] == 'deb' or args['deb_tag']:
        print 'Create output directory: %s/cling-%s-1'%(workdir, VERSION)
        os.makedirs('%s/cling-%s-1'%(workdir, VERSION))

        for file in glob.glob(r'%s/cling_%s*'%(workdir, VERSION)):
            print file + '->' + '%s/cling-%s-1/%s'%(workdir, VERSION, os.path.basename(file))
            shutil.move(file, '%s/cling-%s-1'%(workdir, VERSION))

        if not os.listdir('%s/cling-%s-1'%(workdir, VERSION)):
            os.rmdir('%s/cling-%s-1'%(workdir, VERSION))

###############################################################################
#            Debian specific functions (ported from debianize.sh)             #
###############################################################################

SIGNING_USER = exec_subprocess_check_output('gpg --fingerprint | grep uid | sed s/"uid *"//g', CLING_SRC_DIR).strip()

def tarball_deb():
    box_draw("Compress compiled binaries into a bzip2 tarball")
    tar = tarfile.open(os.path.join(workdir, 'cling_' + VERSION +'.orig.tar.bz2'), 'w:bz2')
    tar.add(prefix, arcname=os.path.basename(prefix))
    tar.close()

def debianize():
    box_draw("Set up the debian directory")
    print "Create directory: debian"
    os.makedirs(os.path.join(prefix, 'debian'))

    print "Create directory: " + os.path.join(prefix, 'debian', 'source')
    os.makedirs(os.path.join(prefix, 'debian', 'source'))

    print "Create file: " + os.path.join(prefix, 'debian', 'source', 'format')
    f=open(os.path.join(prefix, 'debian', 'source', 'format'), 'w')
    f.write('3.0 (quilt)')
    f.close()

    print "Create file: " + os.path.join(prefix, 'debian', 'source', 'lintian-overrides')
    f=open(os.path.join(prefix, 'debian', 'source', 'lintian-overrides'), 'w')
    f.write('cling source: source-is-missing')
    f.close()<commit_msg>Split commands containing quotes into proper list for cmd.exe<commit_after>#! /usr/bin/env python
# coding:utf-8

###############################################################################
#
#                           The Cling Interpreter
#
# Cling Packaging Tool (CPT)
#
# tools/packaging/cpt.py: Main script which calls other helper scripts to
# compile and package Cling for multiple platforms.
#
# TODO: Add documentation here, or provide link to documentation
#
# Author: <NAME> <<EMAIL>>
#
# This file is dual-licensed: you can choose to license it under the University
# of Illinois Open Source License or the GNU Lesser General Public License. See
# LICENSE.TXT for details.
#
###############################################################################

import argparse
import urllib2
import os
import sys
import platform
import subprocess
import shutil
import glob
import re
import tarfile
from email.utils import formatdate
from datetime import tzinfo
import time
import multiprocessing
import fileinput

###############################################################################
#                           Platform initialization                           #
###############################################################################

OS=platform.system()
FAMILY=os.name.upper()

if OS == 'Windows':
    DIST = 'N/A'
    RELEASE = OS + ' ' + platform.release()
    REV = platform.version()

    EXEEXT = '.exe'
    SHLIBEXT = '.dll'

    TMP_PREFIX='C:\\Windows\\Temp\\cling-obj'
    workdir = 'C:\\ec\\build'

elif OS == 'Linux':
    DIST = platform.linux_distribution()[0]
    RELEASE = platform.linux_distribution()[2]
    REV = platform.linux_distribution()[1]

    EXEEXT = ''
    SHLIBEXT = '.so'

    TMP_PREFIX='/var/tmp/cling-obj/'
    workdir = os.path.expanduser('~/ec/build')

elif OS == 'Darwin':
    DIST = 'N/A'
    RELEASE = platform.release()
    REV = platform.mac_ver()[0]

    EXEEXT = ''
    SHLIBEXT = '.dylib'

    TMP_PREFIX='/var/tmp/cling-obj/'
    workdir = os.path.expanduser('~/ec/build')

else:
    # Extensions will be detected anyway by set_ext()
    EXEEXT = ''
    SHLIBEXT = ''

    #TODO: Need to test this in other platforms
    TMP_PREFIX='/var/tmp/cling-obj/'
    workdir = os.path.expanduser('~/ec/build')


###############################################################################
#                               Global variables                              #
###############################################################################

srcdir = os.path.join(workdir, 'cling-src')
CLING_SRC_DIR = os.path.join(srcdir, 'tools', 'cling')
LLVM_OBJ_ROOT = os.path.join(workdir, 'builddir')
prefix = ''
LLVM_GIT_URL = 'http://root.cern.ch/git/llvm.git'
CLANG_GIT_URL = 'http://root.cern.ch/git/clang.git'
CLING_GIT_URL = 'http://root.cern.ch/git/cling.git'
LLVMRevision = urllib2.urlopen("https://raw.githubusercontent.com/ani07nov/cling/master/LastKnownGoodLLVMSVNRevision.txt").readline().strip()
VERSION=''


###############################################################################
#              Platform independent functions (formerly indep.py)             #
###############################################################################

def exec_subprocess_call(cmd, cwd):
    if OS == 'Windows':
        if '"' not in cmd:
            subprocess.Popen(cmd.split(),
                             cwd=cwd,
                             shell=True,
                             stdin=subprocess.PIPE,
                             stdout=None,
                             stderr=subprocess.STDOUT).communicate()
        else:
            subprocess.Popen(cmd.split('"')[0].split() + [cmd.split('"')[1]] + cmd.split('"')[2].split(),
                             cwd=cwd,
                             shell=True,
                             stdin=subprocess.PIPE,
                             stdout=None,
                             stderr=subprocess.STDOUT).communicate()
    else:
        subprocess.Popen([cmd],
                         cwd=cwd,
                         shell=True,
                         stdin=subprocess.PIPE,
                         stdout=None,
                         stderr=subprocess.STDOUT).communicate()

def exec_subprocess_check_output(cmd, cwd):
    if OS == 'Windows':
        return subprocess.Popen(cmd.split(),
                     cwd=cwd,
                     shell=True,
                     stdin=subprocess.PIPE,
                     stdout=subprocess.PIPE,
                     stderr=subprocess.STDOUT).communicate()[0]
    else:
        return subprocess.Popen([cmd],
                     cwd=cwd,
                     shell=True,
                     stdin=subprocess.PIPE,
                     stdout=subprocess.PIPE,
                     stderr=subprocess.STDOUT).communicate()[0]


def box_draw_header():
    msg='cling (' + platform.machine() + ')' + formatdate(time.time(),tzinfo())
    spaces_no = 80 - len(msg) - 4
    spacer = ' ' * spaces_no
    msg='cling (' + platform.machine() + ')' + spacer + formatdate(time.time(),tzinfo())

    if OS != 'Windows':
        print '''
╔══════════════════════════════════════════════════════════════════════════════╗
║ %s ║
╚══════════════════════════════════════════════════════════════════════════════╝'''%(msg)
    else:
        print '''
+=============================================================================+
| %s|
+=============================================================================+'''%(msg)


def box_draw(msg):
    spaces_no = 80 - len(msg) - 4
    spacer = ' ' * spaces_no

    if OS != 'Windows':
        print '''
┌──────────────────────────────────────────────────────────────────────────────┐
│ %s%s │
└──────────────────────────────────────────────────────────────────────────────┘'''%(msg, spacer)
    else:
        print '''
+-----------------------------------------------------------------------------+
| %s%s|
+-----------------------------------------------------------------------------+'''%(msg, spacer)


def fetch_llvm():
    box_draw("Fetch source files")
    print "Last known good LLVM revision is: %s"%(LLVMRevision)
    def get_fresh_llvm():
        exec_subprocess_call('git clone %s %s'%(LLVM_GIT_URL, srcdir), workdir)

        exec_subprocess_call('git checkout ROOT-patches-r%s'%(LLVMRevision), srcdir)

    def update_old_llvm():
        exec_subprocess_call('git stash', srcdir)

        exec_subprocess_call('git clean -f -x -d', srcdir)

        exec_subprocess_call('git fetch --tags', srcdir)

        exec_subprocess_call('git checkout ROOT-patches-r%s'%(LLVMRevision), srcdir)

        exec_subprocess_call('git pull origin refs/tags/ROOT-patches-r%s'%(LLVMRevision), srcdir)

    if os.path.isdir(srcdir):
        update_old_llvm()
    else:
        get_fresh_llvm()


def fetch_clang():
    def get_fresh_clang():
        exec_subprocess_call('git clone %s'%(CLANG_GIT_URL), os.path.join(srcdir, 'tools'))

        exec_subprocess_call('git checkout ROOT-patches-r%s'%(LLVMRevision), os.path.join(srcdir, 'tools', 'clang'))

    def update_old_clang():
        exec_subprocess_call('git stash', os.path.join(srcdir, 'tools', 'clang'))

        exec_subprocess_call('git clean -f -x -d', os.path.join(srcdir, 'tools', 'clang'))

        exec_subprocess_call('git fetch --tags', os.path.join(srcdir, 'tools', 'clang'))

        exec_subprocess_call('git checkout ROOT-patches-r%s'%(LLVMRevision), os.path.join(srcdir, 'tools', 'clang'))

        exec_subprocess_call('git pull origin refs/tags/ROOT-patches-r%s'%(LLVMRevision), os.path.join(srcdir, 'tools', 'clang'))

    if os.path.isdir(os.path.join(srcdir, 'tools', 'clang')):
        update_old_clang()
    else:
        get_fresh_clang()

def fetch_cling(arg):
    def get_fresh_cling():
        exec_subprocess_call('git clone %s'%(CLING_GIT_URL), os.path.join(srcdir, 'tools'))

        if arg == 'last-stable':
            checkout_branch = exec_subprocess_check_output('git describe --match v* --abbrev=0 --tags | head -n 1', CLING_SRC_DIR)

        elif arg == 'master':
            checkout_branch = 'master'
        else:
            checkout_branch = arg

        exec_subprocess_call('git checkout %s'%(checkout_branch), CLING_SRC_DIR)

    def update_old_cling():
        exec_subprocess_call('git stash', CLING_SRC_DIR)

        exec_subprocess_call('git clean -f -x -d', CLING_SRC_DIR)

        exec_subprocess_call('git fetch --tags', CLING_SRC_DIR)

        if arg == 'last-stable':
            checkout_branch = exec_subprocess_check_output('git describe --match v* --abbrev=0 --tags | head -n 1', CLING_SRC_DIR)

        elif arg == 'master':
            checkout_branch = 'master'
        else:
            checkout_branch = arg

        exec_subprocess_call('git checkout %s'%(checkout_branch), CLING_SRC_DIR)

        exec_subprocess_call('git pull origin %s'%(checkout_branch), CLING_SRC_DIR)

    if os.path.isdir(CLING_SRC_DIR):
        update_old_cling()
    else:
        get_fresh_cling()


def set_version():
    global VERSION
    box_draw("Set Cling version")
    VERSION=open(CLING_SRC_DIR+'/VERSION', 'r').readline().strip()

    # If development release, then add revision to the version
    REVISION = exec_subprocess_check_output('git log -n 1 --pretty=format:%H', CLING_SRC_DIR).strip()

    if '~dev' in VERSION:
        VERSION = VERSION + '-' + REVISION[:7]

    print 'Version: ' + VERSION
    print 'Revision: ' + REVISION


def set_ext():
    global EXEEXT
    global SHLIBEXT
    box_draw("Set binary/library extensions")
    if not os.path.isfile(os.path.join(LLVM_OBJ_ROOT, 'test', 'lit.site.cfg')):
        exec_subprocess_call('make lit.site.cfg', os.path.join(LLVM_OBJ_ROOT, 'test'))

    with open(LLVM_OBJ_ROOT + '/test/lit.site.cfg', 'r') as lit_site_cfg:
        for line in lit_site_cfg:
            if re.match('^config.llvm_shlib_ext = ', line):
                SHLIBEXT = re.sub('^config.llvm_shlib_ext = ', '', line).replace('"', '').strip()
            elif re.match('^config.llvm_exe_ext = ', line):
                EXEEXT = re.sub('^config.llvm_exe_ext = ', '', line).replace('"', '').strip()

    print 'EXEEXT: ' + EXEEXT
    print 'SHLIBEXT: ' + SHLIBEXT


def compile(arg):
    global prefix
    prefix=arg
    python=sys.executable
    cores=multiprocessing.cpu_count()

    # Cleanup previous installation directory if any
    if os.path.isdir(prefix):
        print "Remove directory: " + prefix
        shutil.rmtree(prefix)

    # Cleanup previous build directory if exists
    if os.path.isdir(os.path.join(workdir, 'builddir')):
        print "Remove directory: " + os.path.join(workdir, 'builddir')
        shutil.rmtree(os.path.join(workdir, 'builddir'))

    os.makedirs(os.path.join(workdir, 'builddir'))

    if platform.system() == 'Windows':
        box_draw("Configure Cling with CMake and generate Visual Studio 11 project files")
        exec_subprocess_call('cmake -G "Visual Studio 11" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=%s ..\%s'%(TMP_PREFIX, os.path.basename(srcdir)), LLVM_OBJ_ROOT)

        box_draw("Building Cling (using %s cores)"%(cores))
        exec_subprocess_call('cmake --build . --target clang --config Release', LLVM_OBJ_ROOT)

        exec_subprocess_call('cmake --build . --target cling --config Release', LLVM_OBJ_ROOT)

        box_draw("Install compiled binaries to prefix (using %s cores)"%(cores))
        exec_subprocess_call('cmake --build . --target INSTALL --config Release', LLVM_OBJ_ROOT)

    else:
        box_draw("Configure Cling with GNU Make")
        exec_subprocess_call('%s/configure --disable-compiler-version-checks --with-python=%s --enable-targets=host --prefix=%s --enable-optimized=yes --enable-cxx11'%(srcdir, python, TMP_PREFIX), LLVM_OBJ_ROOT)

        box_draw("Building Cling (using %s cores)"%(cores))
        exec_subprocess_call('make -j%s'%(cores), LLVM_OBJ_ROOT)

        box_draw("Install compiled binaries to prefix (using %s cores)"%(cores))
        exec_subprocess_call('make install -j%s prefix=%s'%(cores, TMP_PREFIX), LLVM_OBJ_ROOT)


def install_prefix():
    set_ext()
    box_draw("Filtering Cling's libraries and binaries")

    for line in fileinput.input(CLING_SRC_DIR + '/tools/packaging/dist-files.mk', inplace=True):
        if '@EXEEXT@' in line:
            print line.replace('@EXEEXT@', EXEEXT),
        elif '@SHLIBEXT@' in line:
            print line.replace('@SHLIBEXT@', SHLIBEXT),
        else:
            print line,

    dist_files = open(CLING_SRC_DIR + '/tools/packaging/dist-files.mk', 'r').read()
    for root, dirs, files in os.walk(TMP_PREFIX):
        for file in files:
            f=os.path.join(root, file).replace(TMP_PREFIX, '')
            if f+' ' in dist_files:
                print "Filter: " + f
                if not os.path.isdir(os.path.join(prefix,os.path.dirname(f))):
		    os.makedirs(os.path.join(prefix,os.path.dirname(f)))
                shutil.copy(os.path.join(TMP_PREFIX,f), os.path.join(prefix,f))

def test_cling():
    box_draw("Run Cling test suite")
    if platform.system() != 'Windows':
        exec_subprocess_call('make test', workdir + '/builddir/tools/cling')


def tarball():
    box_draw("Compress binaries into a bzip2 tarball")
    tar = tarfile.open(prefix+'.tar.bz2', 'w:bz2')
    tar.add(prefix, arcname=os.path.basename(prefix))
    tar.close()


def cleanup():
    print "\n"
    box_draw("Clean up")
    if os.path.isdir(os.path.join(workdir,'builddir')):
        print "Remove directory: " + os.path.join(workdir,'builddir')
        shutil.rmtree(os.path.join(workdir,'builddir'))

    if os.path.isdir(prefix):
        print "Remove directory: " + prefix
        shutil.rmtree(prefix)

    if os.path.isdir(TMP_PREFIX):
        print "Remove directory: " + TMP_PREFIX
        shutil.rmtree(TMP_PREFIX)

    if os.path.isfile(os.path.join(workdir,'cling.nsi')):
        print "Remove file: " + os.path.join(workdir,'cling.nsi')
        os.remove(os.path.join(workdir,'cling.nsi'))

    if args['current_dev'] == 'deb' or args['last_stable'] == 'deb' or args['deb_tag']:
        print 'Create output directory: %s/cling-%s-1'%(workdir, VERSION)
        os.makedirs('%s/cling-%s-1'%(workdir, VERSION))

        for file in glob.glob(r'%s/cling_%s*'%(workdir, VERSION)):
            print file + '->' + '%s/cling-%s-1/%s'%(workdir, VERSION, os.path.basename(file))
            shutil.move(file, '%s/cling-%s-1'%(workdir, VERSION))

        if not os.listdir('%s/cling-%s-1'%(workdir, VERSION)):
            os.rmdir('%s/cling-%s-1'%(workdir, VERSION))

###############################################################################
#            Debian specific functions (ported from debianize.sh)             #
###############################################################################

def tarball_deb():
    box_draw("Compress compiled binaries into a bzip2 tarball")
    tar = tarfile.open(os.path.join(workdir, 'cling_' + VERSION +'.orig.tar.bz2'), 'w:bz2')
    tar.add(prefix, arcname=os.path.basename(prefix))
    tar.close()

def debianize():
    SIGNING_USER = exec_subprocess_check_output('gpg --fingerprint | grep uid | sed s/"uid *"//g', CLING_SRC_DIR).strip()

    box_draw("Set up the debian directory")
    print "Create directory: debian"
    os.makedirs(os.path.join(prefix, 'debian'))

    print "Create directory: " + os.path.join(prefix, 'debian', 'source')
    os.makedirs(os.path.join(prefix, 'debian', 'source'))

    print "Create file: " + os.path.join(prefix, 'debian', 'source', 'format')
    f=open(os.path.join(prefix, 'debian', 'source', 'format'), 'w')
    f.write('3.0 (quilt)')
    f.close()

    print "Create file: " + os.path.join(prefix, 'debian', 'source', 'lintian-overrides')
    f=open(os.path.join(prefix, 'debian', 'source', 'lintian-overrides'), 'w')
    f.write('cling source: source-is-missing')
    f.close()
{'fraction_non_alphanumeric': 0.22259347653142403, 'fraction_numerical': 0.002832140015910899, 'mean_word_length': 2.922366450324513, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5437509', 'n_tokens_mistral': 10474, 'n_tokens_neox': 9772, 'n_words': 1940}