<commit_before>from __future__ import absolute_import
# Implementation of RAKE - Rapid Automtic Keyword Exraction algorithm
# as described in:
# <NAME>., <NAME>, <NAME>, and <NAME> (2010).
# Automatic keyword extraction from indi-vidual documents.
# In <NAME> and <NAME> (Eds.), Text Mining: Applications and Theory.unknown: John Wiley and Sons, Ltd.

import re
import operator


def is_number(s):
    try:
        float(s) if '.' in s else int(s)
        return True
    except ValueError:
        return False


def SmartStopList():
    from .stoplists import SmartStopList
    return SmartStopList.words()


def FoxStopList():
    from .stoplists import FoxStopList
    return FoxStopList.words()


def MySQLStopList():
    from .stoplists import MySQLStopList
    return MySQLStopList.words()


def NLTKStopList():
    from .stoplists import NLTKStopList
    return NLTKStopList.words()


def load_stop_words(stop_word_file, regex):
    with open(stop_word_file) as stop_word_file:<commit_msg>adds new lists because i somehow forgot<commit_after>from __future__ import absolute_import
# Implementation of RAKE - Rapid Automtic Keyword Exraction algorithm
# as described in:
# <NAME>., <NAME>, <NAME>, and <NAME> (2010).
# Automatic keyword extraction from indi-vidual documents.
# In <NAME> and <NAME> (Eds.), Text Mining: Applications and Theory.unknown: John Wiley and Sons, Ltd.

import re
import operator


def is_number(s):
    try:
        float(s) if '.' in s else int(s)
        return True
    except ValueError:
        return False


def SmartStopList():
    from .stoplists import SmartStopList
    return SmartStopList.words()


def FoxStopList():
    from .stoplists import FoxStopList
    return FoxStopList.words()


def MySQLStopList():
    from .stoplists import MySQLStopList
    return MySQLStopList.words()


def NLTKStopList():
    from .stoplists import NLTKStopList
    return NLTKStopList.words()

def GoogleSearchStopList():
    from .stoplists import GoogleSearchStopList
    return GoogleSearchStopList.words()

def RanksNLLongStopList():
    from .stoplists import RanksNLLongStopList
    return RanksNLLongStopList.words()

def RanksNLStoplist():
    from .stoplists import RanksNLStoplist
    return RanksNLStoplist.words()

def load_stop_words(stop_word_file, regex):
    with open(stop_word_file) as stop_word_file:
{'fraction_non_alphanumeric': 0.08290816326530612, 'fraction_numerical': 0.003401360544217687, 'mean_word_length': 3.9957537154989384, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '560453', 'n_tokens_mistral': 736, 'n_tokens_neox': 705, 'n_words': 229}