<commit_before>class ParseWarning(Warning):
    def __init__(self, value, **kwargs):
        self.value = value
        self.kwargs = kwargs

class TitleParser(object):
    
    qualities = ['1080p', '1080', '720p', '720', 'hr', 'dvd', 'dvdrip', 'hdtv', 
                 'pdtv', 'dsr', 'dsrip', 'unknown']
    
    propers = ['proper', 'repack', 'rerip']
    
    specials = ['special']
    
    cutoffs = ['dvdrip', 'dvdscr', 'cam', 'r5', 'limited', 'xvid', 'h264', 
               'x264', 'h.264', 'x.264', 'screener', 'unrated', 
               'bluray', 'multisubs'] + propers + specials + qualities
    
    remove = ['imax']

    def strip_spaces(self, text):
        """Removes all unnecessary duplicate spaces from a text"""
        s = text.strip()
        while s.find('  ') != -1:
            s = s.replace('  ', ' ')

        return s<commit_msg>Movie parser improvement<commit_after>class ParseWarning(Warning):
    def __init__(self, value, **kwargs):
        self.value = value
        self.kwargs = kwargs

class TitleParser(object):
    
    qualities = ['1080p', '1080', '720p', '720', 'hr', 'dvd', 'dvdrip', 'hdtv', 
                 'pdtv', 'dsr', 'dsrip', 'unknown']
    
    propers = ['proper', 'repack', 'rerip']
    
    specials = ['special']
    
    cutoffs = ['dvdrip', 'dvdscr', 'cam', 'r5', 'limited', 'xvid', 'h264', 
               'x264', 'h.264', 'x.264', 'screener', 'unrated', '3d', 
               'bluray', 'multisubs'] + propers + specials + qualities
    
    remove = ['imax']

    def strip_spaces(self, text):
        """Removes all unnecessary duplicate spaces from a text"""
        s = text.strip()
        while s.find('  ') != -1:
            s = s.replace('  ', ' ')

        return s
{'fraction_non_alphanumeric': 0.18074986010072747, 'fraction_numerical': 0.0318970341354225, 'mean_word_length': 2.0721649484536084, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4097425', 'n_tokens_mistral': 660, 'n_tokens_neox': 578, 'n_words': 144}