<commit_before># -*- coding: utf-8 -*-
# Install notebook extensions

from __future__ import print_function
from jupyter_core.paths import jupyter_data_dir
import errno
import os
import sys
import shutil
import tempfile
import IPython
import notebook

debug = False

bom_pref = 'ipython-contrib-IPython-notebook-extensions-'
bom_path = os.path.join(jupyter_data_dir(), bom_pref + 'installed_files.txt')

if IPython.__version__[0] < '4':
    print("IPython version 4.x is required")
    exit(1)

if notebook.__version__[0] < '4':
    print("notebook version 4.x is required")
    exit(1)

if (len(sys.argv) == 2 and sys.argv[1] == "debug") or '--debug' in sys.argv:
    debug = True


# http://stackoverflow.com/questions/12683834/how-to-copy-directory-recursively-in-python-and-overwrite-all
def recursive_overwrite(src, dest, ignore=None):
    destinations = []
    if os.path.isdir(src):
        if not os.path.isdir(dest):
            os.makedirs(dest)
        files = os.listdir(src)
        if ignore is not None:
            ignored = ignore(src, files)
        else:
            ignored = set()
        for f in files:
            if f not in ignored:
                destinations.extend(
                    recursive_overwrite(
                        os.path.join(src, f), os.path.join(dest, f), ignore)
                )
    else:
        shutil.copyfile(src, dest)
        destinations.append(dest)
        if debug:
            print('    ' + dest)
    return destinations


def uninstall_files():
    """
    Remove any files recorded as being installed by a previous installation"""

    if os.path.exists(bom_path):
        # rather than actually deleting, copy everything to a temporary
        # directory without explicit cleanup
        deleted_to = tempfile.mkdtemp(prefix=bom_pref)
        print('Removing previously-installed files to {}'.format(deleted_to))
        with open(bom_path, 'r') as bom_file:
            for src in bom_file.readlines():
                src = src.rstrip('\n').rstrip('\r')
                if os.path.exists(src):
                    if debug:
                        print('    ' + src)
                    dest = os.path.join(
                        deleted_to, os.path.relpath(src, data_dir))
                    dest_dir = os.path.dirname(dest)
                    if not os.path.exists(dest_dir):
                        os.makedirs(dest_dir)
                    shutil.move(src, dest)
                # remove empty directories
                while len(src) > len(data_dir):
                    src = os.path.dirname(src)
                    try:
                        os.rmdir(src)
                    except FileNotFoundError:
                        break
                    except OSError as ex:
                        if ex.errno not in (errno.ENOTDIR, errno.ENOTEMPTY):
                            raise
                        break
                    else:
                        if debug:
                            print('    ' + src)
        os.remove(bom_path)

#
# 1. Get the data directory, which will be in jupyter_path
#
data_dir = jupyter_data_dir()

print("Extensions and templates path: %s" % data_dir)
<commit_msg>FileNotFoundError is not available in Python 2<commit_after># -*- coding: utf-8 -*-
# Install notebook extensions

from __future__ import print_function
from jupyter_core.paths import jupyter_data_dir
import errno
import os
import sys
import shutil
import tempfile
import IPython
import notebook

debug = False

bom_pref = 'ipython-contrib-IPython-notebook-extensions-'
bom_path = os.path.join(jupyter_data_dir(), bom_pref + 'installed_files.txt')

if IPython.__version__[0] < '4':
    print("IPython version 4.x is required")
    exit(1)

if notebook.__version__[0] < '4':
    print("notebook version 4.x is required")
    exit(1)

if (len(sys.argv) == 2 and sys.argv[1] == "debug") or '--debug' in sys.argv:
    debug = True


# http://stackoverflow.com/questions/12683834/how-to-copy-directory-recursively-in-python-and-overwrite-all
def recursive_overwrite(src, dest, ignore=None):
    destinations = []
    if os.path.isdir(src):
        if not os.path.isdir(dest):
            os.makedirs(dest)
        files = os.listdir(src)
        if ignore is not None:
            ignored = ignore(src, files)
        else:
            ignored = set()
        for f in files:
            if f not in ignored:
                destinations.extend(
                    recursive_overwrite(
                        os.path.join(src, f), os.path.join(dest, f), ignore)
                )
    else:
        shutil.copyfile(src, dest)
        destinations.append(dest)
        if debug:
            print('    ' + dest)
    return destinations


def uninstall_files():
    """
    Remove any files recorded as being installed by a previous installation"""

    if os.path.exists(bom_path):
        # rather than actually deleting, copy everything to a temporary
        # directory without explicit cleanup
        deleted_to = tempfile.mkdtemp(prefix=bom_pref)
        print('Removing previously-installed files to {}'.format(deleted_to))
        with open(bom_path, 'r') as bom_file:
            for src in bom_file.readlines():
                src = src.rstrip('\n').rstrip('\r')
                if os.path.exists(src):
                    if debug:
                        print('    ' + src)
                    dest = os.path.join(
                        deleted_to, os.path.relpath(src, data_dir))
                    dest_dir = os.path.dirname(dest)
                    if not os.path.exists(dest_dir):
                        os.makedirs(dest_dir)
                    shutil.move(src, dest)
                # remove empty directories
                while len(src) > len(data_dir):
                    src = os.path.dirname(src)
                    try:
                        os.rmdir(src)
                    except OSError as ex:
                        if ex.errno not in (errno.ENOTDIR, errno.ENOTEMPTY):
                            raise
                        break
                    else:
                        if debug:
                            print('    ' + src)
        os.remove(bom_path)

#
# 1. Get the data directory, which will be in jupyter_path
#
data_dir = jupyter_data_dir()

print("Extensions and templates path: %s" % data_dir)

{'fraction_non_alphanumeric': 0.09831681610822715, 'fraction_numerical': 0.006449583136699701, 'mean_word_length': 1.7078364565587734, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4495023', 'n_tokens_mistral': 1900, 'n_tokens_neox': 1818, 'n_words': 465}