<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-


"""
======================================
Configure IPython Notebook Environment
======================================

:Author:
    <NAME>
:Date:
    2011-12-31
:Copyright:
    Copyright(c) 2011 Jacobs University of Bremen. All rights reserved.
:File:
    notebooks.py
"""


import logging
import sys
import os
import shutil
import pwd
import grp
import subprocess
import errno
import string
import random
import csv
import ConfigParser
import time
import codecs
import socket
import pexpect
# non-standard but must for ipython notebook
import tornado.ioloop
import tornado.web
import tornado.template

#from IPython.utils.path import get_ipython_dir
from IPython.lib import passwd


################################################################################
# Utility
################################################################################


LOGGER = logging.getLogger()
#LOGGER.setLevel(logging.WARN)
LOGGER.setLevel(logging.DEBUG)
LOGGER.addHandler(logging.StreamHandler())


# expected header for the user database file
FIELDNAMES = ["name", "surname", "username", "email", "sys-pass", "nb-pass", "port"]


def read_database(filename, enc="utf-8"):
    """
    Reads a spreadsheet-like file with user information.

    Parameters
    ----------
    filename: `str`
        System path that specifies the file location.
    enc: `str` (optional)
        The encoding of the specified file.

    Warnings
    --------
    The `csv` reader may have issues with UTF-16 encoded files.
    """
    global FIELDNAMES
    sniffer = csv.Sniffer()
    with codecs.open(filename, "rb", encoding=enc) as file_handle:
        local_dialect = sniffer.sniff(file_handle.readline(), delimiters=u",;:\t")
        file_handle.seek(0)
        reader = csv.DictReader(file_handle, dialect=local_dialect)
        if set(FIELDNAMES).issubset(set(reader.fieldnames)):
            FIELDNAMES = reader.fieldnames
        else:
            raise ValueError(u"database file lacks required field names:\n"\
                    u"\theader format should be '%s'" % u",".join(FIELDNAMES))
        users = [row for row in reader]
    return (users, local_dialect)

def write_database(filename, users, local_dialect="excel", enc="utf-8"):
    """
    Writes a spreadsheet-like file with user information.

    Parameters
    ----------
    filename: `str`
        System path that specifies where the file should be written.
    users: `list`
        List whose elements are one `dict` per user. Keywords correspond to
        field names and values are entries.
    local_dialect: `csv.Dialect` or `str` (optional)
        Can be a `csv.Dialect` instance or a string specifying the dialect used.
    enc: `str` (optional)
        The encoding of the specified file.

    Warnings
    --------
    If a user `dict` contains a key that is not in the standard header, writing
    will fail and all file contents except for the header are lost.
    """
    with codecs.open(filename, "wb", encoding=enc) as file_handle:
        writer = csv.DictWriter(file_handle, FIELDNAMES, dialect=local_dialect)
        writer.writeheader()
        writer.writerows(users)

def parse_config(filename, enc="utf-8"):
    """
    Handles parsing of the configuration file.

    Parameters
    ----------
    filename: `str`
        System path that specifies the file location.
    enc: `str` (optional)
        The encoding of the specified file.
    """

    def get_local_ip():
        """
        This method works unreliably.
        """
        return [ip for ip in socket.gethostbyname_ex(socket.gethostname())[2]\
                if not ip.startswith("127.")]

    def get_network_ip():
        """
        Retrieve your network IP by opening a socket to google.

        May not find your public IP if you're behind NAT but often the network<commit_msg>replaced writeheader to support Python 2.6<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-


"""
======================================
Configure IPython Notebook Environment
======================================

:Author:
    <NAME>
:Date:
    2011-12-31
:Copyright:
    Copyright(c) 2011 Jacobs University of Bremen. All rights reserved.
:File:
    notebooks.py
"""


import logging
import sys
import os
import shutil
import pwd
import grp
import subprocess
import errno
import string
import random
import csv
import ConfigParser
import time
import codecs
import socket
import pexpect
# non-standard but must for ipython notebook
import tornado.ioloop
import tornado.web
import tornado.template

#from IPython.utils.path import get_ipython_dir
from IPython.lib import passwd


################################################################################
# Utility
################################################################################


LOGGER = logging.getLogger()
#LOGGER.setLevel(logging.WARN)
LOGGER.setLevel(logging.DEBUG)
LOGGER.addHandler(logging.StreamHandler())


# expected header for the user database file
FIELDNAMES = ["name", "surname", "username", "email", "sys-pass", "nb-pass", "port"]


def read_database(filename, enc="utf-8"):
    """
    Reads a spreadsheet-like file with user information.

    Parameters
    ----------
    filename: `str`
        System path that specifies the file location.
    enc: `str` (optional)
        The encoding of the specified file.

    Warnings
    --------
    The `csv` reader may have issues with UTF-16 encoded files.
    """
    global FIELDNAMES
    sniffer = csv.Sniffer()
    with codecs.open(filename, "rb", encoding=enc) as file_handle:
        local_dialect = sniffer.sniff(file_handle.readline(), delimiters=u",;:\t")
        file_handle.seek(0)
        reader = csv.DictReader(file_handle, dialect=local_dialect)
        if set(FIELDNAMES).issubset(set(reader.fieldnames)):
            FIELDNAMES = reader.fieldnames
        else:
            raise ValueError(u"database file lacks required field names:\n"\
                    u"\theader format should be '%s'" % u",".join(FIELDNAMES))
        users = [row for row in reader]
    return (users, local_dialect)

def write_database(filename, users, local_dialect="excel", enc="utf-8"):
    """
    Writes a spreadsheet-like file with user information.

    Parameters
    ----------
    filename: `str`
        System path that specifies where the file should be written.
    users: `list`
        List whose elements are one `dict` per user. Keywords correspond to
        field names and values are entries.
    local_dialect: `csv.Dialect` or `str` (optional)
        Can be a `csv.Dialect` instance or a string specifying the dialect used.
    enc: `str` (optional)
        The encoding of the specified file.

    Warnings
    --------
    If a user `dict` contains a key that is not in the standard header, writing
    will fail and all file contents except for the header are lost.
    """
    with codecs.open(filename, "wb", encoding=enc) as file_handle:
        writer = csv.DictWriter(file_handle, FIELDNAMES, dialect=local_dialect)
        writer.writerow(dict(zip(FIELDNAMES, FIELDNAMES)))
        writer.writerows(users)

def parse_config(filename, enc="utf-8"):
    """
    Handles parsing of the configuration file.

    Parameters
    ----------
    filename: `str`
        System path that specifies the file location.
    enc: `str` (optional)
        The encoding of the specified file.
    """

    def get_local_ip():
        """
        This method works unreliably.
        """
        return [ip for ip in socket.gethostbyname_ex(socket.gethostname())[2]\
                if not ip.startswith("127.")]

    def get_network_ip():
        """
        Retrieve your network IP by opening a socket to google.

        May not find your public IP if you're behind NAT but often the network
{'fraction_non_alphanumeric': 0.15819640810087887, 'fraction_numerical': 0.00611387084447841, 'mean_word_length': 3.0557851239669422, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5643548', 'n_tokens_mistral': 2259, 'n_tokens_neox': 2171, 'n_words': 725}