<commit_before>""" Common tools used by plugins implementing search plugin api """
from __future__ import unicode_literals, division, absolute_import
from builtins import *

import re
from unicodedata import normalize

from flexget.utils.titles.parser import TitleParser


def clean_symbols(text):
    """Replaces common symbols with spaces. Also normalize unicode strings in decomposed form."""
    result = text
    if isinstance(result, str):
        result = normalize('NFKD', result)
    result = re.sub('[ \(\)\-_\[\]\.]+', ' ', result).lower()

	# Leftovers
    result = re.sub(r"[^a-zA-Z0-9 ]", "", result)

    return result


def clean_title(title):
    """Removes common codec, sound keywords, and special characters info from titles to facilitate
    loose title comparison.
    """
    result = TitleParser.remove_words(title, TitleParser.sounds + TitleParser.codecs)
    result = clean_symbols(result)
    return result

<commit_msg>Remove tab character from search.py<commit_after>""" Common tools used by plugins implementing search plugin api """
from __future__ import unicode_literals, division, absolute_import
from builtins import *

import re
from unicodedata import normalize

from flexget.utils.titles.parser import TitleParser


def clean_symbols(text):
    """Replaces common symbols with spaces. Also normalize unicode strings in decomposed form."""
    result = text
    if isinstance(result, str):
        result = normalize('NFKD', result)
    result = re.sub('[ \(\)\-_\[\]\.]+', ' ', result).lower()

    # Leftovers
    result = re.sub(r"[^a-zA-Z0-9 ]", "", result)

    return result


def clean_title(title):
    """Removes common codec, sound keywords, and special characters info from titles to facilitate
    loose title comparison.
    """
    result = TitleParser.remove_words(title, TitleParser.sounds + TitleParser.codecs)
    result = clean_symbols(result)
    return result


{'fraction_non_alphanumeric': 0.10995310057321521, 'fraction_numerical': 0.0020844189682126106, 'mean_word_length': 3.9104859335038364, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6576738', 'n_tokens_mistral': 557, 'n_tokens_neox': 525, 'n_words': 176}