<commit_before>import urllib
import urllib2
import urlparse
import logging
import re
import string
import difflib

# this way we don't force users to install bs incase they do not want to use this module
soup_present = True
soup_err = "Module filter_imdb requires BeautifulSoup. Please install it from http://www.crummy.com/software/BeautifulSoup/ or from your distribution repository."

try:
    from BeautifulSoup import BeautifulSoup
except:
    log.warning(soup_err)
    soup_present = False

log = logging.getLogger('imdb')

class ImdbSearch:

    def __init__(self):
        # depriorize aka matches a bit
        self.aka_weight = 0.9
        self.min_match = 0.5
        self.min_diff = 0.01
        self.debug = True
        self.cutoffs = ['dvdrip', 'dvdscr', 'cam', 'r5', 'limited',
                        'xvid', 'h264', 'x264', 'dvd', 'screener',
                        'unrated', 'repack', 'rerip']
        self.ignore_types = ['VG']

    def parse_name(self, s):
        """Sanitizes movie name from all kind of crap"""
        # todo: improve, remove hacks  ..
        s = s.replace('h.264', 'h264')
        s = s.replace('x.264', 'x264')
        s = s.replace('.', ' ')
        s = s.replace('_', ' ')
        s = s.replace('-', ' ')
        parts = s.split(' ')
        year = None
        cut_pos = 256<commit_msg>Added even more imdb search cutwords<commit_after>import urllib
import urllib2
import urlparse
import logging
import re
import string
import difflib

# this way we don't force users to install bs incase they do not want to use this module
soup_present = True
soup_err = "Module filter_imdb requires BeautifulSoup. Please install it from http://www.crummy.com/software/BeautifulSoup/ or from your distribution repository."

try:
    from BeautifulSoup import BeautifulSoup
except:
    log.warning(soup_err)
    soup_present = False

log = logging.getLogger('imdb')

class ImdbSearch:

    def __init__(self):
        # depriorize aka matches a bit
        self.aka_weight = 0.9
        self.min_match = 0.5
        self.min_diff = 0.01
        self.debug = True
        self.cutoffs = ['dvdrip', 'dvdscr', 'cam', 'r5', 'limited',
                        'xvid', 'h264', 'x264', 'dvd', 'screener',
                        'unrated', 'repack', 'rerip', 'proper']
        self.ignore_types = ['VG']

    def parse_name(self, s):
        """Sanitizes movie name from all kind of crap"""
        # todo: improve, remove hacks  ..
        s = s.replace('h.264', 'h264')
        s = s.replace('x.264', 'x264')
        s = s.replace('.', ' ')
        s = s.replace('_', ' ')
        s = s.replace('-', ' ')
        parts = s.split(' ')
        year = None
        cut_pos = 256
{'fraction_non_alphanumeric': 0.12467628560858306, 'fraction_numerical': 0.022197558268590455, 'mean_word_length': 2.375780274656679, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4083173', 'n_tokens_mistral': 914, 'n_tokens_neox': 856, 'n_words': 257}