<commit_before>#! /usr/bin/env python
# coding:utf-8

###############################################################################
#
#                           The Cling Interpreter
#
# Cling Packaging Tool (CPT)
#
# tools/packaging/cpt.py: Python script to launch Cling Packaging Tool (CPT)
#
# Documentation: tools/packaging/README.md
#
# Author: <NAME> <<EMAIL>>
#
# This file is dual-licensed: you can choose to license it under the University
# of Illinois Open Source License or the GNU Lesser General Public License. See
# LICENSE.TXT for details.
#
###############################################################################

# Python 2 and Python 3 compatibility
from __future__ import print_function

import sys

if sys.version_info < (3, 0):
    # Python 2.x
    from urllib2 import urlopen

    input = raw_input
else:
    # Python 3.x
    from urllib.request import urlopen

import argparse
import os
import platform
import subprocess
import shutil
import shlex
import glob
import re
import tarfile
import zipfile
from email.utils import formatdate
from datetime import tzinfo
import time
import pip
import multiprocessing
import fileinput
import stat
import json


###############################################################################
#              Platform independent functions (formerly indep.py)             #
###############################################################################

def _convert_subprocess_cmd(cmd):
    if OS == 'Windows':
        if '"' in cmd:
            # Assume there's only one quoted argument.
            bits = cmd.split('"')
            return bits[0].split() + [bits[1]] + bits[2].split()
        else:
            return cmd.split()
    else:
        return shlex.split(cmd, posix=True, comments=True)

def _pdebug_info():
    box_draw("Printing debug information")

    print("Printing CMakeOutput.log")
    with open(os.path.join(LLVM_OBJ_ROOT, 'CMakeFiles', 'CMakeOutput.log'), 'r') as f:
        print(f.read())

    print("Printing CMakeError.log")
    with open(os.path.join(LLVM_OBJ_ROOT, 'CMakeFiles', 'CMakeError.log'), 'r') as f:
        print(f.read())


def _perror(e):
    print("subprocess.CalledProcessError: Command '%s' returned non-zero exit status %s" % (
        ' '.join(e.cmd), str(e.returncode)))
    # Print out some useful information, while keeping all files around in --debug-on-error.
    if args['debug_on_error']:
        _pdebug_info()
    else:
        cleanup()
    # Communicate return code to the calling program if any
    sys.exit(e.returncode)


def exec_subprocess_call(cmd, cwd):
    cmd = _convert_subprocess_cmd(cmd)
    try:
        subprocess.check_call(cmd, cwd=cwd, shell=False,
                              stdin=subprocess.PIPE, stdout=None, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        _perror(e)


def exec_subprocess_check_output(cmd, cwd):
    cmd = _convert_subprocess_cmd(cmd)
    try:
        out = subprocess.check_output(cmd, cwd=cwd, shell=False,
                                      stdin=subprocess.PIPE, stderr=subprocess.STDOUT).decode('utf-8')
    except subprocess.CalledProcessError as e:
        _perror(e)

    finally:
        return out


def box_draw_header():
    msg = 'cling (' + platform.machine() + ')' + formatdate(time.time(), tzinfo())
    spaces_no = 80 - len(msg) - 4
    spacer = ' ' * spaces_no
    msg = 'cling (' + platform.machine() + ')' + spacer + formatdate(time.time(), tzinfo())

    if OS != 'Windows':
        print('''
╔══════════════════════════════════════════════════════════════════════════════╗
║ %s ║
╚══════════════════════════════════════════════════════════════════════════════╝''' % (msg))
    else:
        print('''
+=============================================================================+
| %s|
+=============================================================================+''' % (msg))


def box_draw(msg):
    spaces_no = 80 - len(msg) - 4
    spacer = ' ' * spaces_no

    if OS != 'Windows':
        print('''
┌──────────────────────────────────────────────────────────────────────────────┐
│ %s%s │
└──────────────────────────────────────────────────────────────────────────────┘''' % (msg, spacer))
    else:
        print('''
+-----------------------------------------------------------------------------+
| %s%s|
+-----------------------------------------------------------------------------+''' % (msg, spacer))

def pip_install(package):
    pip.main(['install', '--ignore-installed', '--prefix', os.path.join(workdir, 'pip'), '--upgrade', package])


def wget(url, out_dir, rename_file=None, retries=3):
    file_name = url.split('/')[-1]
    print("  HTTP request sent, awaiting response ... ")
    u = urlopen(url)
    if u.code != 200 or retries == 0:
        exit()
    else:
        print("  Connected to %s [200 OK]" % (url))

    try:
        file_size = u.headers.get('Content-Length')
        if file_size:
            file_size = int(file_size)
        else:
            raise Exception
    except Exception:
        print('  Error due to broken pipe')
        print('  Retrying ...')
        wget(url, out_dir, retries-1)

    else:
        print("  Downloading: %s Bytes: %s" % (file_name, file_size))<commit_msg>Only import pip if needed.<commit_after>#! /usr/bin/env python
# coding:utf-8

###############################################################################
#
#                           The Cling Interpreter
#
# Cling Packaging Tool (CPT)
#
# tools/packaging/cpt.py: Python script to launch Cling Packaging Tool (CPT)
#
# Documentation: tools/packaging/README.md
#
# Author: <NAME> <<EMAIL>>
#
# This file is dual-licensed: you can choose to license it under the University
# of Illinois Open Source License or the GNU Lesser General Public License. See
# LICENSE.TXT for details.
#
###############################################################################

# Python 2 and Python 3 compatibility
from __future__ import print_function

import sys

if sys.version_info < (3, 0):
    # Python 2.x
    from urllib2 import urlopen

    input = raw_input
else:
    # Python 3.x
    from urllib.request import urlopen

import argparse
import os
import platform
import subprocess
import shutil
import shlex
import glob
import re
import tarfile
import zipfile
from email.utils import formatdate
from datetime import tzinfo
import time
import multiprocessing
import fileinput
import stat
import json


###############################################################################
#              Platform independent functions (formerly indep.py)             #
###############################################################################

def _convert_subprocess_cmd(cmd):
    if OS == 'Windows':
        if '"' in cmd:
            # Assume there's only one quoted argument.
            bits = cmd.split('"')
            return bits[0].split() + [bits[1]] + bits[2].split()
        else:
            return cmd.split()
    else:
        return shlex.split(cmd, posix=True, comments=True)

def _pdebug_info():
    box_draw("Printing debug information")

    print("Printing CMakeOutput.log")
    with open(os.path.join(LLVM_OBJ_ROOT, 'CMakeFiles', 'CMakeOutput.log'), 'r') as f:
        print(f.read())

    print("Printing CMakeError.log")
    with open(os.path.join(LLVM_OBJ_ROOT, 'CMakeFiles', 'CMakeError.log'), 'r') as f:
        print(f.read())


def _perror(e):
    print("subprocess.CalledProcessError: Command '%s' returned non-zero exit status %s" % (
        ' '.join(e.cmd), str(e.returncode)))
    # Print out some useful information, while keeping all files around in --debug-on-error.
    if args['debug_on_error']:
        _pdebug_info()
    else:
        cleanup()
    # Communicate return code to the calling program if any
    sys.exit(e.returncode)


def exec_subprocess_call(cmd, cwd):
    cmd = _convert_subprocess_cmd(cmd)
    try:
        subprocess.check_call(cmd, cwd=cwd, shell=False,
                              stdin=subprocess.PIPE, stdout=None, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        _perror(e)


def exec_subprocess_check_output(cmd, cwd):
    cmd = _convert_subprocess_cmd(cmd)
    try:
        out = subprocess.check_output(cmd, cwd=cwd, shell=False,
                                      stdin=subprocess.PIPE, stderr=subprocess.STDOUT).decode('utf-8')
    except subprocess.CalledProcessError as e:
        _perror(e)

    finally:
        return out


def box_draw_header():
    msg = 'cling (' + platform.machine() + ')' + formatdate(time.time(), tzinfo())
    spaces_no = 80 - len(msg) - 4
    spacer = ' ' * spaces_no
    msg = 'cling (' + platform.machine() + ')' + spacer + formatdate(time.time(), tzinfo())

    if OS != 'Windows':
        print('''
╔══════════════════════════════════════════════════════════════════════════════╗
║ %s ║
╚══════════════════════════════════════════════════════════════════════════════╝''' % (msg))
    else:
        print('''
+=============================================================================+
| %s|
+=============================================================================+''' % (msg))


def box_draw(msg):
    spaces_no = 80 - len(msg) - 4
    spacer = ' ' * spaces_no

    if OS != 'Windows':
        print('''
┌──────────────────────────────────────────────────────────────────────────────┐
│ %s%s │
└──────────────────────────────────────────────────────────────────────────────┘''' % (msg, spacer))
    else:
        print('''
+-----------------------------------------------------------------------------+
| %s%s|
+-----------------------------------------------------------------------------+''' % (msg, spacer))

def pip_install(package):
    # Needs brew install python. We should only install if we need the
    # functionality
    import pip
    pip.main(['install', '--ignore-installed', '--prefix', os.path.join(workdir, 'pip'), '--upgrade', package])


def wget(url, out_dir, rename_file=None, retries=3):
    file_name = url.split('/')[-1]
    print("  HTTP request sent, awaiting response ... ")
    u = urlopen(url)
    if u.code != 200 or retries == 0:
        exit()
    else:
        print("  Connected to %s [200 OK]" % (url))

    try:
        file_size = u.headers.get('Content-Length')
        if file_size:
            file_size = int(file_size)
        else:
            raise Exception
    except Exception:
        print('  Error due to broken pipe')
        print('  Retrying ...')
        wget(url, out_dir, retries-1)

    else:
        print("  Downloading: %s Bytes: %s" % (file_name, file_size))
{'fraction_non_alphanumeric': 0.28410159924741296, 'fraction_numerical': 0.005268109125117592, 'mean_word_length': 2.946176688938382, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3478209', 'n_tokens_mistral': 3337, 'n_tokens_neox': 3042, 'n_words': 788}