<commit_before># coding: utf-8

import requests
import os
import json
import subprocess
import shutil
import sys
import zipfile

"""
Setup environment for run test
"""

TMP_FOLDER = os.sep.join([os.getcwd(), 'install'])


def head():
    """Print head"""

    title = """\n
    #####################################################
    # Setup environment for run tests.                  #
    # Install:                                          #
    # - chromedriver                                    #
    # - selenium-standalone-server                      #
    # - test                                            #
    # - Page Objects                                    #
    # - lib selenium                                    #
    #####################################################
    """

    print(title)


def svn_checkout(url, path):
    """Call svn checkout"""
    user = config()['SVN_USERNAME']
    password = config()['SVN_PASSWORD']
    p = subprocess.Popen(['svn', 'co', url, path, '--username', user, '--password', password])
    stdout, stderr = p.communicate()


def download_file(url, file_name):
    """Download file from remote host"""

    r = requests.get(url=url, stream=True)
    if r.ok:
        with open(os.path.join(TMP_FOLDER, file_name), 'wb') as f:
            f.write(r.content)
    else:
        r.raise_for_status()


def config():
    """Work with config file"""

    with open('config.json', encoding='utf-8') as conf:
        json_conf = json.load(conf)
    return json_conf


def get_chromedriver():
    """Get chromedriver from internet"""

    ver_chromedriver = config()['VER_CHROMEDRIVER']
    win_filename = 'chromedriver_win32.zip'
    url_chromedriver = 'http://chromedriver.storage.googleapis.com/{}/{}'.format(ver_chromedriver, win_filename)
    download_file(url_chromedriver, win_filename)


def get_standalone_server():
    """Get selenium-standalone-server.jar"""

    ver_standalone_server = config()['VER_STANDALONE_SERVER']
    patch = '1'
    file_standalone_server = 'selenium-server-standalone-{ver}.{p}.jar'.format(ver=ver_standalone_server,
                                                                               p=patch or '0')
    url_selenium_server = 'http://selenium-release.storage.googleapis.com/{}/{}'.format(ver_standalone_server,
                                                                                        file_standalone_server)
    download_file(url_selenium_server, file_standalone_server)


def get_atf():
    """Get automated testing framework"""
    ver_atf = config()['VER_ATF']
    url = 'https://svn.sbis.ru/svn/sbis3-tests/atf/release/{}/'.format(ver_atf)
    path = os.path.join(TMP_FOLDER, 'atf')
    if os.path.exists(path):
        shutil.rmtree(path)
    svn_checkout(url, path)


def get_genie_tests():
    """Get test from svn"""

    ver_genie = config()['VER_GENIE']
    url = 'https://svn.sbis.ru/svn/sbis3-tests/genie/{}/'.format(ver_genie)
    path = os.path.join(TMP_FOLDER, 'tests')
    if os.path.exists(path):
        shutil.rmtree(path)
    svn_checkout(url, path)


def install_lib():
    """Install lib's through pip"""

    p = subprocess.Popen(['pip', 'install', '-r', 'requirement.txt'])
    p.communicate()


def copy_into_site_packages():
    """Copy atf, PO into folder site-packages"""

    site_packages = [path for path in sys.path if 'site-packages' in path][0]
    try:
        shutil.copytree(os.path.join(TMP_FOLDER, 'atf'), os.path.join(site_packages, 'atf'))
        shutil.copytree(os.path.join(TMP_FOLDER, 'tests', 'module_genie'), os.path.join(site_packages, 'module_genie'))
        shutil.copytree(os.path.join(TMP_FOLDER, 'tests', 'pages'), os.path.join(site_packages, 'pages'))
    except FileExistsError:
        pass


def copy_chromedriver():
    """Copy chromedriver in execute path"""

    exec_dir = sys.prefix
    zipfile.ZipFile(os.path.join(TMP_FOLDER, 'chromedriver_win32.zip')).extractall(path=os.path.join(exec_dir, 'bin'))<commit_msg>change requests lib to urllib<commit_after># coding: utf-8

import urllib.request as request
import os
import json
import subprocess
import shutil
import sys
import zipfile
import argparse

"""
Setup environment for run test
"""

TMP_FOLDER = os.sep.join([os.getcwd(), 'install'])


def head():
    """Print head"""

    title = """\n
    #####################################################
    # Setup environment for run tests.                  #
    # Install:                                          #
    # - chromedriver                                    #
    # - selenium-standalone-server                      #
    # - test                                            #
    # - Page Objects                                    #
    # - lib selenium                                    #
    #####################################################
    """

    print(title)


def svn_checkout(url, path):
    """Call svn checkout"""
    user = config()['SVN_USERNAME']
    password = config()['SVN_PASSWORD']
    exit_code = subprocess.Popen(['svn', 'co', url, path, '--username', user, '--password', password]).wait()
    if exit_code != 0:
        raise Exception('Don\'t get data from repository')


def download_file(url, file_name):
    """Download file from remote host"""

    request.urlretrieve(url=url, filename=os.path.join(TMP_FOLDER, file_name))


def config():
    """Work with config file"""

    with open('config.json', encoding='utf-8') as conf:
        json_conf = json.load(conf)
    return json_conf


def get_chromedriver():
    """Get chromedriver from internet"""

    ver_chromedriver = config()['VER_CHROMEDRIVER']
    win_filename = 'chromedriver_win32.zip'
    url_chromedriver = 'http://chromedriver.storage.googleapis.com/{}/{}'.format(ver_chromedriver, win_filename)
    download_file(url_chromedriver, win_filename)


def get_standalone_server():
    """Get selenium-standalone-server.jar"""

    ver_standalone_server = config()['VER_STANDALONE_SERVER']
    patch = '1'
    file_standalone_server = 'selenium-server-standalone-{ver}.{p}.jar'.format(ver=ver_standalone_server,
                                                                               p=patch or '0')
    url_selenium_server = 'http://selenium-release.storage.googleapis.com/{}/{}'.format(ver_standalone_server,
                                                                                        file_standalone_server)
    download_file(url_selenium_server, file_standalone_server)


def get_atf():
    """Get automated testing framework"""
    ver_atf = config()['VER_ATF']
    url = 'https://svn.sbis.ru/svn/sbis3-tests/atf/release/{}/'.format(ver_atf)
    path = os.path.join(TMP_FOLDER, 'atf')
    if os.path.exists(path):
        shutil.rmtree(path)
    svn_checkout(url, path)


def get_genie_tests():
    """Get test from svn"""

    ver_genie = config()['VER_GENIE']
    url = 'https://svn.sbis.ru/svn/sbis3-tests/genie/{}/'.format(ver_genie)
    path = os.path.join(TMP_FOLDER, 'tests')
    if os.path.exists(path):
        shutil.rmtree(path)
    svn_checkout(url, path)


def install_lib():
    """Install lib's through pip"""

    user = config()['SVN_USERNAME']
    password = config()['SVN_PASSWORD']
    os.environ['HTTP_PROXY'] = 'http://{}:{}@ias.corp.tensor.ru:8080'.format(user, password)
    p = subprocess.Popen(['pip', 'install', '-r', 'requirement.txt'])
    p.communicate()


def copy_into_site_packages():
    """Copy atf, PO into folder site-packages"""

    site_packages = [path for path in sys.path if 'site-packages' in path][0]
    try:
        shutil.copytree(os.path.join(TMP_FOLDER, 'atf'), os.path.join(site_packages, 'atf'))
        shutil.copytree(os.path.join(TMP_FOLDER, 'tests', 'module_genie'), os.path.join(site_packages, 'module_genie'))
        shutil.copytree(os.path.join(TMP_FOLDER, 'tests', 'pages'), os.path.join(site_packages, 'pages'))
    except FileExistsError:
        pass


def copy_chromedriver():
    """Copy chromedriver in execute path"""

    exec_dir = sys.prefix
    zipfile.ZipFile(os.path.join(TMP_FOLDER, 'chromedriver_win32.zip')).extractall(path=os.path.join(exec_dir, 'bin'))
{'fraction_non_alphanumeric': 0.16352047070360382, 'fraction_numerical': 0.003309634714390782, 'mean_word_length': 2.562882096069869, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7075078', 'n_tokens_mistral': 2626, 'n_tokens_neox': 2525, 'n_words': 475}