<commit_before>from __future__ import absolute_import
# Implementation of RAKE - Rapid Automtic Keyword Exraction algorithm
# as described in:
# <NAME>., <NAME>, <NAME>, and <NAME> (2010).
# Automatic keyword extraction from indi-vidual documents.
# In <NAME> and <NAME> (Eds.), Text Mining: Applications and Theory.unknown: John Wiley and Sons, Ltd.

import re
import operator


def is_number(s):
    try:
        float(s) if '.' in s else int(s)
        return True
    except ValueError:
        return False


def SmartStopList():
    from .stoplists import SmartStopList
    return SmartStopList.words()


def FoxStopList():
    from .stoplists import FoxStopList
    return FoxStopList.words()


def MySQLStopList():
    from .stoplists import MySQLStopList
    return MySQLStopList.words()


def NLTKStopList():
    from .stoplists import NLTKStopList
    return NLTKStopList.words()


def GoogleSearchStopList():
    from .stoplists import GoogleSearchStopList
    return GoogleSearchStopList.words()


def RanksNLLongStopList():
    from .stoplists import RanksNLLongStopList
    return RanksNLLongStopList.words()


def RanksNLStoplist():
    from .stoplists import RanksNLStoplist
    return RanksNLStoplist.words()


def load_stop_words(stop_word_file, regex):
    with open(stop_word_file) as stop_word_file:
        stop_words = re.split(regex, stop_word_file.read())
    return [word for word in stop_words if word not in ('', ' ')]  # filters empty string matches


def separate_words(text):
    """
    Utility function to return a list of all words that are have a length greater than a specified number of characters.
    @param text The text that must be split in to words.
    @param min_word_return_size The minimum no of characters a word must have to be included.
    """
    splitter = re.compile('\W+')
    words = []
    for single_word in splitter.split(text):
        current_word = single_word.strip().lower()
        # leave numbers in phrase, but don't count as words, since they tend to invalidate scores of their phrases
        if current_word != '' and not is_number(current_word):
            words.append(current_word)
    return words


def split_sentences(text):
    """
    Utility function to return a list of sentences.
    @param text The text that must be split in to sentences.
    """
    sentence_delimiters = re.compile('[.!?,;:\t\\\\"\\(\\)\\\'\u2019\u2013]|\\s\\-\\s')<commit_msg>Fix Issue #30 - Split words on U (#31)<commit_after>from __future__ import absolute_import
# Implementation of RAKE - Rapid Automtic Keyword Exraction algorithm
# as described in:
# <NAME>., <NAME>, <NAME>, and <NAME> (2010).
# Automatic keyword extraction from indi-vidual documents.
# In <NAME> and <NAME> (Eds.), Text Mining: Applications and Theory.unknown: <NAME> and Sons, Ltd.

import re
import operator


def is_number(s):
    try:
        float(s) if '.' in s else int(s)
        return True
    except ValueError:
        return False


def SmartStopList():
    from .stoplists import SmartStopList
    return SmartStopList.words()


def FoxStopList():
    from .stoplists import FoxStopList
    return FoxStopList.words()


def MySQLStopList():
    from .stoplists import MySQLStopList
    return MySQLStopList.words()


def NLTKStopList():
    from .stoplists import NLTKStopList
    return NLTKStopList.words()


def GoogleSearchStopList():
    from .stoplists import GoogleSearchStopList
    return GoogleSearchStopList.words()


def RanksNLLongStopList():
    from .stoplists import RanksNLLongStopList
    return RanksNLLongStopList.words()


def RanksNLStoplist():
    from .stoplists import RanksNLStoplist
    return RanksNLStoplist.words()


def load_stop_words(stop_word_file, regex):
    with open(stop_word_file) as stop_word_file:
        stop_words = re.split(regex, stop_word_file.read())
    return [word for word in stop_words if word not in ('', ' ')]  # filters empty string matches


def separate_words(text):
    """
    Utility function to return a list of all words that are have a length greater than a specified number of characters.
    @param text The text that must be split in to words.
    @param min_word_return_size The minimum no of characters a word must have to be included.
    """
    splitter = re.compile('\W+')
    words = []
    for single_word in splitter.split(text):
        current_word = single_word.strip().lower()
        # leave numbers in phrase, but don't count as words, since they tend to invalidate scores of their phrases
        if current_word != '' and not is_number(current_word):
            words.append(current_word)
    return words


def split_sentences(text):
    """
    Utility function to return a list of sentences.
    @param text The text that must be split in to sentences.
    """
    sentence_delimiters = re.compile(u'[.!?,;:\t\\\\"\\(\\)\\\'\u2019\u2013]|\\s\\-\\s')
{'fraction_non_alphanumeric': 0.09479934479934479, 'fraction_numerical': 0.005733005733005733, 'mean_word_length': 3.4128274616079493, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '705112', 'n_tokens_mistral': 1494, 'n_tokens_neox': 1437, 'n_words': 499}