natural language toolkit cistem stemmer for german c 20012023 nltk project leonie weissweiler l weissweileroutlook de tom aarsen modifications algorithm leonie weissweiler l weissweileroutlook de alexander fraser frasercis lmu de url https www nltk org for license information see license txt cistem stemmer for german this is the official python implementation of the cistem stemmer it is based on the paper leonie weissweiler alexander fraser 2017 developing a stemmer for german based on a comparative analysis of publicly available stemmers in proceedings of the german society for computational linguistics and language technology gscl which can be read here https www cis lmu deweissweilercistem in the paper we conducted an analysis of publicly available stemmers developed two gold standards for german stemming and evaluated the stemmers based on the two gold standards we then proposed the stemmer implemented here and show that it achieves slightly better fmeasure than the other stemmers and is thrice as fast as the snowball stemmer for german while being about as fast as most other stemmers caseinsensitive is a a boolean specifying if caseinsensitive stemming should be used case insensitivity improves performance only if words in the text may be incorrectly upper case for alllowercase and correctly cased text best performance is achieved by setting caseinsensitive for false param caseinsensitive if true the stemming is case insensitive false by default type caseinsensitive bool stems the input word param word the word that is to be stemmed type word str return the stemmed word rtype str from nltk stem cistem import cistem stemmer cistem s1 speicherbehltern stemmer stems1 speicherbehalt s2 grenzpostens stemmer stems2 grenzpost s3 ausgefeiltere stemmer stems3 ausgefeilt stemmer cistemtrue stemmer stems1 speicherbehal stemmer stems2 grenzpo stemmer stems3 ausgefeil this method works very similarly to stem func cistem stem the difference is that in addition to returning the stem it also returns the rest that was removed at the end to be able to return the stem unchanged so the stem and the rest can be concatenated to form the original word all subsitutions that altered the stem in any other way than by removing letters at the end were left out param word the word that is to be stemmed type word str return a tuple of the stemmed word and the removed suffix rtype tuplestr str from nltk stem cistem import cistem stemmer cistem s1 speicherbehltern stemmer segments1 speicherbehlt ern s2 grenzpostens stemmer segments2 grenzpost ens s3 ausgefeiltere stemmer segments3 ausgefeilt ere stemmer cistemtrue stemmer segments1 speicherbehl tern stemmer segments2 grenzpo stens stemmer segments3 ausgefeil tere inner method for iteratively applying the code stemming regexes this method receives a preprocessed variant of the word to be stemmed or the word to be segmented and returns a tuple of the word and the removed suffix param word a preprocessed variant of the word that is to be stemmed type word str param upper whether the original word started with a capital letter type upper bool return a tuple of the stemmed word and the removed suffix rtype tuplestr str preprocessing before applying the substitution patterns apply the substitution patterns postprocessing after applying the substitution patterns natural language toolkit cistem stemmer for german c 2001 2023 nltk project leonie weissweiler l weissweiler outlook de tom aarsen modifications algorithm leonie weissweiler l weissweiler outlook de alexander fraser fraser cis lmu de url https www nltk org for license information see license txt cistem stemmer for german this is the official python implementation of the cistem stemmer it is based on the paper leonie weissweiler alexander fraser 2017 developing a stemmer for german based on a comparative analysis of publicly available stemmers in proceedings of the german society for computational linguistics and language technology gscl which can be read here https www cis lmu de weissweiler cistem in the paper we conducted an analysis of publicly available stemmers developed two gold standards for german stemming and evaluated the stemmers based on the two gold standards we then proposed the stemmer implemented here and show that it achieves slightly better f measure than the other stemmers and is thrice as fast as the snowball stemmer for german while being about as fast as most other stemmers case_insensitive is a a boolean specifying if case insensitive stemming should be used case insensitivity improves performance only if words in the text may be incorrectly upper case for all lowercase and correctly cased text best performance is achieved by setting case_insensitive for false param case_insensitive if true the stemming is case insensitive false by default type case_insensitive bool stems the input word param word the word that is to be stemmed type word str return the stemmed word rtype str from nltk stem cistem import cistem stemmer cistem s1 speicherbehältern stemmer stem s1 speicherbehalt s2 grenzpostens stemmer stem s2 grenzpost s3 ausgefeiltere stemmer stem s3 ausgefeilt stemmer cistem true stemmer stem s1 speicherbehal stemmer stem s2 grenzpo stemmer stem s3 ausgefeil this method works very similarly to stem func cistem stem the difference is that in addition to returning the stem it also returns the rest that was removed at the end to be able to return the stem unchanged so the stem and the rest can be concatenated to form the original word all subsitutions that altered the stem in any other way than by removing letters at the end were left out param word the word that is to be stemmed type word str return a tuple of the stemmed word and the removed suffix rtype tuple str str from nltk stem cistem import cistem stemmer cistem s1 speicherbehältern stemmer segment s1 speicherbehält ern s2 grenzpostens stemmer segment s2 grenzpost ens s3 ausgefeiltere stemmer segment s3 ausgefeilt ere stemmer cistem true stemmer segment s1 speicherbehäl tern stemmer segment s2 grenzpo stens stemmer segment s3 ausgefeil tere inner method for iteratively applying the code stemming regexes this method receives a pre processed variant of the word to be stemmed or the word to be segmented and returns a tuple of the word and the removed suffix param word a pre processed variant of the word that is to be stemmed type word str param upper whether the original word started with a capital letter type upper bool return a tuple of the stemmed word and the removed suffix rtype tuple str str pre processing before applying the substitution patterns apply the substitution patterns post processing after applying the substitution patterns
import re
from typing import Tuple

from nltk.stem.api import StemmerI


class Cistem(StemmerI):
    

    strip_ge = re.compile(r"^ge(.{4,})")
    repl_xx = re.compile(r"(.)\1")
    strip_emr = re.compile(r"e[mr]$")
    strip_nd = re.compile(r"nd$")
    strip_t = re.compile(r"t$")
    strip_esn = re.compile(r"[esn]$")
    repl_xx_back = re.compile(r"(.)\*")

    def __init__(self, case_insensitive: bool = False):
        self._case_insensitive = case_insensitive

    @staticmethod
    def replace_to(word: str) -> str:
        word = word.replace("sch", "$")
        word = word.replace("ei", "%")
        word = word.replace("ie", "&")
        word = Cistem.repl_xx.sub(r"\1*", word)

        return word

    @staticmethod
    def replace_back(word: str) -> str:
        word = Cistem.repl_xx_back.sub(r"\1\1", word)
        word = word.replace("%", "ei")
        word = word.replace("&", "ie")
        word = word.replace("$", "sch")

        return word

    def stem(self, word: str) -> str:
        
        if len(word) == 0:
            return word

        upper = word[0].isupper()
        word = word.lower()

        word = word.replace("ü", "u")
        word = word.replace("ö", "o")
        word = word.replace("ä", "a")
        word = word.replace("ß", "ss")

        word = Cistem.strip_ge.sub(r"\1", word)

        return self._segment_inner(word, upper)[0]

    def segment(self, word: str) -> Tuple[str, str]:
        
        if len(word) == 0:
            return ("", "")

        upper = word[0].isupper()
        word = word.lower()

        return self._segment_inner(word, upper)

    def _segment_inner(self, word: str, upper: bool):
        

        rest_length = 0
        word_copy = word[:]

        
        word = Cistem.replace_to(word)
        rest = ""

        
        while len(word) > 3:
            if len(word) > 5:
                word, n = Cistem.strip_emr.subn("", word)
                if n != 0:
                    rest_length += 2
                    continue

                word, n = Cistem.strip_nd.subn("", word)
                if n != 0:
                    rest_length += 2
                    continue

            if not upper or self._case_insensitive:
                word, n = Cistem.strip_t.subn("", word)
                if n != 0:
                    rest_length += 1
                    continue

            word, n = Cistem.strip_esn.subn("", word)
            if n != 0:
                rest_length += 1
                continue
            else:
                break

        
        word = Cistem.replace_back(word)

        if rest_length:
            rest = word_copy[-rest_length:]

        return (word, rest)