<commit_before># Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.

import argparse
import subprocess
import sys
import os
import re
import shutil
import threading
import tempfile


# Set up the file structure
root_dir = tempfile.mkdtemp(prefix='mock_contents')
os.mkdir(os.path.join(root_dir, 'src'))
with open(os.path.join(root_dir, 'src', 'temp.txt'), 'w') as fid:
    fid.write('hello')


KARMA_PORT = 9876



def start_notebook():
    nb_command = [sys.executable, '-m', 'notebook', root_dir, '--no-browser',
                  '--debug',
                  # FIXME: allow-origin=* only required for notebook < 4.3
                  '--NotebookApp.allow_origin="*"',
                  # disable user password:
                  '--NotebookApp.password=',
              ]
    nb_server = subprocess.Popen(nb_command, stderr=subprocess.STDOUT,
                                 stdout=subprocess.PIPE)

    # wait for notebook server to start up
    while 1:
        line = nb_server.stdout.readline().decode('utf-8').strip()
        if not line:
            continue
        print(line)
        if 'Jupyter Notebook is running at:' in line:
            base_url = re.search(r'(http[^\?]+)', line).groups()[0]
            token_match = re.search(r'token\=([^&]+)', line)
            if token_match:
                token = token_match.groups()[0]
            else:
                token = ''
            break

    while 1:
        line = nb_server.stdout.readline().decode('utf-8').strip()
        if not line:
            continue
        print(line)
        if 'Control-C' in line:
            break

    def print_thread():
        while 1:
            line = nb_server.stdout.readline().decode('utf-8').strip()
            if not line:
                continue
            print(line)

    thread = threading.Thread(target=print_thread)
    thread.setDaemon(True)
    thread.start()

    return nb_server, base_url, token


def run_mocha(options, base_url, token):
    mocha_command = ['mocha', '--timeout', '20000', 'build/integration.js',
                     '--baseUrl=%s' % base_url, '--token=%s' % token]<commit_msg>only pass token if present<commit_after># Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.

import argparse
import subprocess
import sys
import os
import re
import shutil
import threading
import tempfile


# Set up the file structure
root_dir = tempfile.mkdtemp(prefix='mock_contents')
os.mkdir(os.path.join(root_dir, 'src'))
with open(os.path.join(root_dir, 'src', 'temp.txt'), 'w') as fid:
    fid.write('hello')


KARMA_PORT = 9876



def start_notebook():
    nb_command = [sys.executable, '-m', 'notebook', root_dir, '--no-browser',
                  '--debug',
                  # FIXME: allow-origin=* only required for notebook < 4.3
                  '--NotebookApp.allow_origin="*"',
                  # disable user password:
                  '--NotebookApp.password=',
              ]
    nb_server = subprocess.Popen(nb_command, stderr=subprocess.STDOUT,
                                 stdout=subprocess.PIPE)

    # wait for notebook server to start up
    while 1:
        line = nb_server.stdout.readline().decode('utf-8').strip()
        if not line:
            continue
        print(line)
        if 'Jupyter Notebook is running at:' in line:
            base_url = re.search(r'(http[^\?]+)', line).groups()[0]
            token_match = re.search(r'token\=([^&]+)', line)
            if token_match:
                token = token_match.groups()[0]
            else:
                token = ''
            break

    while 1:
        line = nb_server.stdout.readline().decode('utf-8').strip()
        if not line:
            continue
        print(line)
        if 'Control-C' in line:
            break

    def print_thread():
        while 1:
            line = nb_server.stdout.readline().decode('utf-8').strip()
            if not line:
                continue
            print(line)

    thread = threading.Thread(target=print_thread)
    thread.setDaemon(True)
    thread.start()

    return nb_server, base_url, token


def run_mocha(options, base_url, token):
    mocha_command = ['mocha', '--timeout', '20000', 'build/integration.js',
                     '--baseUrl=%s' % base_url]
    if token:
        mocha_command.append('--token=%s' % token)
{'fraction_non_alphanumeric': 0.12681488203266789, 'fraction_numerical': 0.008620689655172414, 'mean_word_length': 2.0448895027624308, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3916718', 'n_tokens_mistral': 1383, 'n_tokens_neox': 1312, 'n_words': 321}