<commit_before># Natural Language Toolkit: Stemmers
#
# Copyright (C) 2001-2017 NLTK Project
# Author: <NAME> <<EMAIL>>
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT

"""
A word stemmer based on the Lancaster stemming algorithm.
Paice, <NAME>. "Another Stemmer." ACM SIGIR Forum 24.3 (1990): 56-61.
"""
from __future__ import unicode_literals
import re

from nltk.stem.api import StemmerI
from nltk.compat import python_2_unicode_compatible

@python_2_unicode_compatible
class LancasterStemmer(StemmerI):
    """
    Lancaster Stemmer

        >>> from nltk.stem.lancaster import LancasterStemmer
        >>> st = LancasterStemmer()
        >>> st.stem('maximum')     # Remove "-um" when word is intact
        'maxim'
        >>> st.stem('presumably')  # Don't remove "-um" when word is not intact
        'presum'
        >>> st.stem('multiply')    # No action taken if word ends with "-ply"
        'multiply'
        >>> st.stem('provision')   # Replace "-sion" with "-j" to trigger "j" set of rules
        'provid'
        >>> st.stem('owed')        # Word starting with vowel must contain at least 2 letters
        'ow'
        >>> st.stem('ear')         # ditto
        'ear'
        >>> st.stem('saying')      # Words starting with consonant must contain at least 3
        'say'
        >>> st.stem('crying')      #     letters and one of those letters must be a vowel
        'cry'<commit_msg>include Paice/Husk in lancaster.py's docstring for Googleability<commit_after># Natural Language Toolkit: Stemmers
#
# Copyright (C) 2001-2017 NLTK Project
# Author: <NAME> <<EMAIL>>
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT

"""
A word stemmer based on the Lancaster (Paice/Husk) stemming algorithm.
Paice, <NAME>. "Another Stemmer." ACM SIGIR Forum 24.3 (1990): 56-61.
"""
from __future__ import unicode_literals
import re

from nltk.stem.api import StemmerI
from nltk.compat import python_2_unicode_compatible

@python_2_unicode_compatible
class LancasterStemmer(StemmerI):
    """
    Lancaster Stemmer

        >>> from nltk.stem.lancaster import LancasterStemmer
        >>> st = LancasterStemmer()
        >>> st.stem('maximum')     # Remove "-um" when word is intact
        'maxim'
        >>> st.stem('presumably')  # Don't remove "-um" when word is not intact
        'presum'
        >>> st.stem('multiply')    # No action taken if word ends with "-ply"
        'multiply'
        >>> st.stem('provision')   # Replace "-sion" with "-j" to trigger "j" set of rules
        'provid'
        >>> st.stem('owed')        # Word starting with vowel must contain at least 2 letters
        'ow'
        >>> st.stem('ear')         # ditto
        'ear'
        >>> st.stem('saying')      # Words starting with consonant must contain at least 3
        'say'
        >>> st.stem('crying')      #     letters and one of those letters must be a vowel
        'cry'
{'fraction_non_alphanumeric': 0.1232876712328767, 'fraction_numerical': 0.015753424657534248, 'mean_word_length': 2.6649937264742785, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3181971', 'n_tokens_mistral': 987, 'n_tokens_neox': 892, 'n_words': 290}