<commit_before>[[analysis-tokenfilters]]
== Token Filters

Token filters accept a stream of tokens from a
<<analysis-tokenizers,tokenizer>> and can modify tokens
(eg lowercasing), delete tokens (eg remove stopwords)
or add tokens (eg synonyms).

Elasticsearch has a number of built in token filters which can be
used to build <<analysis-custom-analyzer,custom analyzers>>.

include::tokenfilters/standard-tokenfilter.asciidoc[]

include::tokenfilters/asciifolding-tokenfilter.asciidoc[]

include::tokenfilters/flatten-graph-tokenfilter.asciidoc[]

include::tokenfilters/length-tokenfilter.asciidoc[]

include::tokenfilters/lowercase-tokenfilter.asciidoc[]

include::tokenfilters/uppercase-tokenfilter.asciidoc[]

include::tokenfilters/ngram-tokenfilter.asciidoc[]

include::tokenfilters/edgengram-tokenfilter.asciidoc[]

include::tokenfilters/porterstem-tokenfilter.asciidoc[]

include::tokenfilters/shingle-tokenfilter.asciidoc[]

include::tokenfilters/stop-tokenfilter.asciidoc[]

include::tokenfilters/word-delimiter-tokenfilter.asciidoc[]

include::tokenfilters/stemmer-tokenfilter.asciidoc[]

include::tokenfilters/stemmer-override-tokenfilter.asciidoc[]

include::tokenfilters/keyword-marker-tokenfilter.asciidoc[]

include::tokenfilters/keyword-repeat-tokenfilter.asciidoc[]
<commit_msg>Add missing link for the WordDelimiterGraphFilter<commit_after>[[analysis-tokenfilters]]
== Token Filters

Token filters accept a stream of tokens from a
<<analysis-tokenizers,tokenizer>> and can modify tokens
(eg lowercasing), delete tokens (eg remove stopwords)
or add tokens (eg synonyms).

Elasticsearch has a number of built in token filters which can be
used to build <<analysis-custom-analyzer,custom analyzers>>.

include::tokenfilters/standard-tokenfilter.asciidoc[]

include::tokenfilters/asciifolding-tokenfilter.asciidoc[]

include::tokenfilters/flatten-graph-tokenfilter.asciidoc[]

include::tokenfilters/length-tokenfilter.asciidoc[]

include::tokenfilters/lowercase-tokenfilter.asciidoc[]

include::tokenfilters/uppercase-tokenfilter.asciidoc[]

include::tokenfilters/ngram-tokenfilter.asciidoc[]

include::tokenfilters/edgengram-tokenfilter.asciidoc[]

include::tokenfilters/porterstem-tokenfilter.asciidoc[]

include::tokenfilters/shingle-tokenfilter.asciidoc[]

include::tokenfilters/stop-tokenfilter.asciidoc[]

include::tokenfilters/word-delimiter-tokenfilter.asciidoc[]

include::tokenfilters/word-delimiter-graph-tokenfilter.asciidoc[]

include::tokenfilters/stemmer-tokenfilter.asciidoc[]

include::tokenfilters/stemmer-override-tokenfilter.asciidoc[]

include::tokenfilters/keyword-marker-tokenfilter.asciidoc[]

include::tokenfilters/keyword-repeat-tokenfilter.asciidoc[]

{'fraction_non_alphanumeric': 0.11408398364920104, 'fraction_numerical': 0.0, 'mean_word_length': 8.187713310580206, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5554315', 'n_tokens_mistral': 801, 'n_tokens_neox': 832, 'n_words': 130}