<commit_before>import codecs
import sqlite3
import re
from fuzzywuzzy import fuzz

# common words in movies that we don't want to search the database for
common_words = [
            "The",
            "Them",
            "A",
            "An",
            ]

# blacklist of common garbage that fills up movie names
blacklist = [
            "BluRay",
            "\d{3,4}p",
            "(HD|DVD)Rip",
            "x\d{3}",
            "XViD(-.*)?",
            ]

# compile the blacklist into a regex
bl_re = re.compile("(" + "|".join(blacklist) + ")(\s|$)", re.IGNORECASE)

conn = sqlite3.connect("movies.db")
c = conn.cursor()

# Setup the sqlite database
def search(query):
    # remove all instancer of 'WORD ' for WORD in blacklist
    query = query.replace(".", " ")
    query = bl_re.sub("", query)

    m = re.search("(\d{4})", query)
    for match in m.groups():
        if match != "1080":
            year = match
<commit_msg>Fix the year finding method.<commit_after>import codecs
import sqlite3
import re
from fuzzywuzzy import fuzz

# common words in movies that we don't want to search the database for
common_words = [
            "The",
            "Them",
            "A",
            "An",
            ]

# blacklist of common garbage that fills up movie names
blacklist = [
            "BluRay",
            "\d{3,4}p",
            "(HD|DVD)Rip",
            "x\d{3}",
            "XViD(-.*)?",
            ]

# compile the blacklist into a regex
bl_re = re.compile("(" + "|".join(blacklist) + ")(\s|$)", re.IGNORECASE)

conn = sqlite3.connect("movies.db")
c = conn.cursor()

# Setup the sqlite database
def search(query):
    # remove all instancer of 'WORD ' for WORD in blacklist
    query = query.replace(".", " ")
    query = bl_re.sub("", query)

    year = re.search("(19|20)\d{2}", query).group(0)

{'fraction_non_alphanumeric': 0.14824945295404815, 'fraction_numerical': 0.011487964989059081, 'mean_word_length': 2.1052631578947367, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3028396', 'n_tokens_mistral': 613, 'n_tokens_neox': 587, 'n_words': 175}