<commit_before>from __future__ import print_function
import re
from pkg_resources import resource_stream
import collections
import cmudict

__author__ = '<NAME>'
__email__ = '<EMAIL>'
__version__ = '0.1.5'

pronunciations = None
lookup = None
rhyme_lookup = None


def parse_cmu(cmufh):
    """Parses an incoming file handle as a CMU pronouncing dictionary file.

    (Most end-users of this module won't need to call this function explicitly,
    as it's called internally by the :func:`init_cmu` function.)

    :param cmufh: a filehandle with CMUdict-formatted data
    :returns: a list of 2-tuples pairing a word with its phones (as a string)
    """
    pronunciations = list()
    for line in cmufh:
        line = line.strip().decode('utf-8')
        if line.startswith(';'):
            continue
        word, phones = line.split(" ", maxsplit=1)
        pronunciations.append((word.split('(', 1)[0].lower(), phones))
    return pronunciations


def init_cmu(filehandle=None):
    """Initialize the module's pronunciation data.

    This function is called automatically the first time you attempt to use
    another function in the library that requires loading the pronunciation
    data from disk. You can call this function manually to control when and<commit_msg>Fixed split to work with 2.7 as well.<commit_after>from __future__ import print_function
import re
from pkg_resources import resource_stream
import collections
import cmudict

__author__ = '<NAME>'
__email__ = '<EMAIL>'
__version__ = '0.1.5'

pronunciations = None
lookup = None
rhyme_lookup = None


def parse_cmu(cmufh):
    """Parses an incoming file handle as a CMU pronouncing dictionary file.

    (Most end-users of this module won't need to call this function explicitly,
    as it's called internally by the :func:`init_cmu` function.)

    :param cmufh: a filehandle with CMUdict-formatted data
    :returns: a list of 2-tuples pairing a word with its phones (as a string)
    """
    pronunciations = list()
    for line in cmufh:
        line = line.strip().decode('utf-8')
        if line.startswith(';'):
            continue
        word, phones = line.split(" ", 1)
        pronunciations.append((word.split('(', 1)[0].lower(), phones))
    return pronunciations


def init_cmu(filehandle=None):
    """Initialize the module's pronunciation data.

    This function is called automatically the first time you attempt to use
    another function in the library that requires loading the pronunciation
    data from disk. You can call this function manually to control when and
{'fraction_non_alphanumeric': 0.08453447604207245, 'fraction_numerical': 0.007012076353720296, 'mean_word_length': 3.6021505376344085, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3272052', 'n_tokens_mistral': 749, 'n_tokens_neox': 720, 'n_words': 290}