<commit_before>#! /usr/bin/env python
# coding:utf-8

###############################################################################
#
#                           The Cling Interpreter
#
# Cling Packaging Tool (CPT)
#
# tools/packaging/cpt.py: Python script to launch Cling Packaging Tool (CPT)
#
# Documentation: tools/packaging/README.md
#
# Author: <NAME> <<EMAIL>>
#
# This file is dual-licensed: you can choose to license it under the University
# of Illinois Open Source License or the GNU Lesser General Public License. See
# LICENSE.TXT for details.
#
###############################################################################

# Python 2 and Python 3 compatibility
from __future__ import print_function

import sys
if sys.version_info < (3,0):
    # Python 2.x
    from urllib2 import urlopen
    input = raw_input
else:
    # Python 3.x
    from urllib.request import urlopen

import argparse
import os
import platform
import subprocess
import shutil
import shlex
import glob
import re
import tarfile
import zipfile
from email.utils import formatdate
from datetime import tzinfo
import time
import multiprocessing
import fileinput
import stat

###############################################################################
#              Platform independent functions (formerly indep.py)             #
###############################################################################

def _convert_subprocess_cmd(cmd):
    if OS == 'Windows':
        if '"' in cmd:
            # Assume there's only one quoted argument.
            bits = cmd.split('"')
            return bits[0].split() + [bits[1]] + bits[2].split()
        else:
            return cmd.split()
    else:
        return [cmd]

def _perror(e):
    print("subprocess.CalledProcessError: Command '%s' returned non-zero exit status %s"%(' '.join(e.cmd), str(e.returncode)))
    cleanup()
    # Communicate return code to the calling program if any
    sys.exit(e.returncode)

def exec_subprocess_call(cmd, cwd):
    cmd = _convert_subprocess_cmd(cmd)
    try:
        subprocess.check_call(cmd, cwd=cwd, shell=True,
                              stdin=subprocess.PIPE, stdout=None, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        _perror(e)

def exec_subprocess_check_output(cmd, cwd):
    cmd = _convert_subprocess_cmd(cmd)
    try:
        out = subprocess.check_output(cmd, cwd=cwd, shell=True,
                                      stdin=subprocess.PIPE, stderr=subprocess.STDOUT).decode('utf-8')
    except subprocess.CalledProcessError as e:
        _perror(e)

    finally:
        return out

def box_draw_header():
    msg='cling (' + platform.machine() + ')' + formatdate(time.time(),tzinfo())
    spaces_no = 80 - len(msg) - 4
    spacer = ' ' * spaces_no
    msg='cling (' + platform.machine() + ')' + spacer + formatdate(time.time(),tzinfo())

    if OS != 'Windows':
        print('''
╔══════════════════════════════════════════════════════════════════════════════╗
║ %s ║
╚══════════════════════════════════════════════════════════════════════════════╝'''%(msg))
    else:
        print('''
+=============================================================================+
| %s|
+=============================================================================+'''%(msg))


def box_draw(msg):
    spaces_no = 80 - len(msg) - 4
    spacer = ' ' * spaces_no

    if OS != 'Windows':
        print('''
┌──────────────────────────────────────────────────────────────────────────────┐
│ %s%s │
└──────────────────────────────────────────────────────────────────────────────┘'''%(msg, spacer))
    else:
        print('''
+-----------------------------------------------------------------------------+
| %s%s|
+-----------------------------------------------------------------------------+'''%(msg, spacer))

def wget(url, out_dir):
    file_name = url.split('/')[-1]
    print("HTTP request sent, awaiting response... ")
    u = urlopen(url)
    if u.code == 200:
        print("Connected to %s [200 OK]"%(url))
    else:
        exit()
    f = open(os.path.join(out_dir, file_name), 'wb')
    if sys.version_info < (3,0):
        # Python 2.x
        meta = u.info()
        file_size = int(meta.getheaders("Content-Length")[0])
    else:
        # Python 3.x
        file_size = int(u.getheader("Content-Length"))

    print("Downloading: %s Bytes: %s" % (file_name, file_size))

    file_size_dl = 0
    block_sz = 8192
    while True:
        buffer = u.read(block_sz)
        if not buffer:
            break

        file_size_dl += len(buffer)
        f.write(buffer)
        status = r"%10d  [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
        status = status + chr(8)*(len(status)+1)
        print(status, end=' ')
    f.close()

def fetch_llvm():
    box_draw("Fetch source files")
    print('Last known good LLVM revision is: ' + LLVMRevision)
    print('Current working directory is: ' + workdir + '\n')
    def get_fresh_llvm():
        exec_subprocess_call('git clone %s %s'%(LLVM_GIT_URL, srcdir), workdir)

        exec_subprocess_call('git checkout cling-patches-r%s'%(LLVMRevision), srcdir)

    def update_old_llvm():
        exec_subprocess_call('git stash', srcdir)

        exec_subprocess_call('git clean -f -x -d', srcdir)

        exec_subprocess_call('git fetch --tags', srcdir)

        exec_subprocess_call('git checkout cling-patches-r%s'%(LLVMRevision), srcdir)

        exec_subprocess_call('git pull origin refs/tags/cling-patches-r%s'%(LLVMRevision), srcdir)

    if os.path.isdir(srcdir):
        update_old_llvm()
    else:
        get_fresh_llvm()


def fetch_clang():
    def get_fresh_clang():
        exec_subprocess_call('git clone %s'%(CLANG_GIT_URL), os.path.join(srcdir, 'tools'))

        exec_subprocess_call('git checkout cling-patches-r%s'%(LLVMRevision), os.path.join(srcdir, 'tools', 'clang'))

    def update_old_clang():<commit_msg>New method to get sources of LLVM and Clang<commit_after>#! /usr/bin/env python
# coding:utf-8

###############################################################################
#
#                           The Cling Interpreter
#
# Cling Packaging Tool (CPT)
#
# tools/packaging/cpt.py: Python script to launch Cling Packaging Tool (CPT)
#
# Documentation: tools/packaging/README.md
#
# Author: <NAME> <<EMAIL>>
#
# This file is dual-licensed: you can choose to license it under the University
# of Illinois Open Source License or the GNU Lesser General Public License. See
# LICENSE.TXT for details.
#
###############################################################################

# Python 2 and Python 3 compatibility
from __future__ import print_function

import sys
if sys.version_info < (3,0):
    # Python 2.x
    from urllib2 import urlopen
    input = raw_input
else:
    # Python 3.x
    from urllib.request import urlopen

import argparse
import os
import platform
import subprocess
import shutil
import shlex
import glob
import re
import tarfile
import zipfile
from email.utils import formatdate
from datetime import tzinfo
import time
import multiprocessing
import fileinput
import stat

###############################################################################
#              Platform independent functions (formerly indep.py)             #
###############################################################################

def _convert_subprocess_cmd(cmd):
    if OS == 'Windows':
        if '"' in cmd:
            # Assume there's only one quoted argument.
            bits = cmd.split('"')
            return bits[0].split() + [bits[1]] + bits[2].split()
        else:
            return cmd.split()
    else:
        return [cmd]

def _perror(e):
    print("subprocess.CalledProcessError: Command '%s' returned non-zero exit status %s"%(' '.join(e.cmd), str(e.returncode)))
    cleanup()
    # Communicate return code to the calling program if any
    sys.exit(e.returncode)

def exec_subprocess_call(cmd, cwd):
    cmd = _convert_subprocess_cmd(cmd)
    try:
        subprocess.check_call(cmd, cwd=cwd, shell=True,
                              stdin=subprocess.PIPE, stdout=None, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        _perror(e)

def exec_subprocess_check_output(cmd, cwd):
    cmd = _convert_subprocess_cmd(cmd)
    try:
        out = subprocess.check_output(cmd, cwd=cwd, shell=True,
                                      stdin=subprocess.PIPE, stderr=subprocess.STDOUT).decode('utf-8')
    except subprocess.CalledProcessError as e:
        _perror(e)

    finally:
        return out

def box_draw_header():
    msg='cling (' + platform.machine() + ')' + formatdate(time.time(),tzinfo())
    spaces_no = 80 - len(msg) - 4
    spacer = ' ' * spaces_no
    msg='cling (' + platform.machine() + ')' + spacer + formatdate(time.time(),tzinfo())

    if OS != 'Windows':
        print('''
╔══════════════════════════════════════════════════════════════════════════════╗
║ %s ║
╚══════════════════════════════════════════════════════════════════════════════╝'''%(msg))
    else:
        print('''
+=============================================================================+
| %s|
+=============================================================================+'''%(msg))


def box_draw(msg):
    spaces_no = 80 - len(msg) - 4
    spacer = ' ' * spaces_no

    if OS != 'Windows':
        print('''
┌──────────────────────────────────────────────────────────────────────────────┐
│ %s%s │
└──────────────────────────────────────────────────────────────────────────────┘'''%(msg, spacer))
    else:
        print('''
+-----------------------------------------------------------------------------+
| %s%s|
+-----------------------------------------------------------------------------+'''%(msg, spacer))

def wget(url, out_dir):
    file_name = url.split('/')[-1]
    print("  HTTP request sent, awaiting response ... ")
    u = urlopen(url)
    if u.code == 200:
        print("  Connected to %s [200 OK]"%(url))
    else:
        exit()

    try:
        if sys.version_info < (3,0):
            # Python 2.x
            meta = u.info()
            file_size = int(meta.getheaders("Content-Length")[0])
        else:
            # Python 3.x
            file_size = int(u.getheader("Content-Length"))
    except IndexError, e:
        print(e)
        print('  Error due to broken pipe. Retrying ...')
        wget(url, out_dir)

    else:
        print("  Downloading: %s Bytes: %s" % (file_name, file_size))

        file_size_dl = 0
        block_sz = 8192

        f = open(os.path.join(out_dir, file_name), 'wb')

        while True:
            buffer = u.read(block_sz)
            if not buffer:
                break

            file_size_dl += len(buffer)
            f.write(buffer)
            status = r"%10d  [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
            status = status + chr(8)*(len(status)+1)
            print(status, end=' ')
        f.close()
        print()

def fetch_llvm():
    box_draw("Fetch source files")
    print('Last known good LLVM revision is: ' + LLVMRevision)
    print('Current working directory is: ' + workdir + '\n')

    if "github.com" in LLVM_GIT_URL and args['create_dev_env'] == None:
        _, _, _, user, repo = LLVM_GIT_URL.split('/')
        print('Fetching LLVM ...')
        wget(url='https://github.com/%s/%s'%(user, repo.replace('.git', '')) + '/archive/cling-patches.tar.gz', out_dir=workdir)

        print('Extracting: ' + os.path.join(workdir, 'cling-patches.tar.gz'))
        tar = tarfile.open(os.path.join(workdir, 'cling-patches.tar.gz'))
        tar.extractall(path=workdir)
        tar.close()

        os.rename(os.path.join(workdir, 'llvm-cling-patches'), srcdir)

        if os.path.isfile(os.path.join(workdir, 'cling-patches.tar.gz')):
            print("Remove file: " + os.path.join(workdir, 'cling-patches.tar.gz'))
            os.remove(os.path.join(workdir, 'cling-patches.tar.gz'))

        print()
        return

    def get_fresh_llvm():
        exec_subprocess_call('git clone %s %s'%(LLVM_GIT_URL, srcdir), workdir)

        exec_subprocess_call('git checkout cling-patches-r%s'%(LLVMRevision), srcdir)

    def update_old_llvm():
        exec_subprocess_call('git stash', srcdir)

        exec_subprocess_call('git clean -f -x -d', srcdir)

        exec_subprocess_call('git fetch --tags', srcdir)

        exec_subprocess_call('git checkout cling-patches-r%s'%(LLVMRevision), srcdir)

        exec_subprocess_call('git pull origin refs/tags/cling-patches-r%s'%(LLVMRevision), srcdir)

    if os.path.isdir(srcdir):
        update_old_llvm()
    else:
        get_fresh_llvm()


def fetch_clang():
    if "github.com" in CLANG_GIT_URL and args['create_dev_env'] == None:
        _, _, _, user, repo = CLANG_GIT_URL.split('/')
        print('Fetching Clang ...')
        wget(url='https://github.com/%s/%s'%(user, repo.replace('.git', '')) + '/archive/cling-patches.tar.gz', out_dir=workdir)

        print('Extracting: ' + os.path.join(workdir, 'cling-patches.tar.gz'))
        tar = tarfile.open(os.path.join(workdir, 'cling-patches.tar.gz'))
        tar.extractall(path=os.path.join(srcdir, 'tools'))
        tar.close()

        os.rename(os.path.join(srcdir, 'tools', 'clang-cling-patches'), os.path.join(srcdir, 'tools', 'clang'))

        if os.path.isfile(os.path.join(workdir, 'cling-patches.tar.gz')):
            print("Remove file: " + os.path.join(workdir, 'cling-patches.tar.gz'))
            os.remove(os.path.join(workdir, 'cling-patches.tar.gz'))

        print()
        return

    def get_fresh_clang():
        exec_subprocess_call('git clone %s'%(CLANG_GIT_URL), os.path.join(srcdir, 'tools'))

        exec_subprocess_call('git checkout cling-patches-r%s'%(LLVMRevision), os.path.join(srcdir, 'tools', 'clang'))

    def update_old_clang():
{'fraction_non_alphanumeric': 0.2571078077644682, 'fraction_numerical': 0.006350122672824361, 'mean_word_length': 2.8973565804274464, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2322570', 'n_tokens_mistral': 4581, 'n_tokens_neox': 4248, 'n_words': 930}