<commit_before>from vectorizor import Vectorizor
import db_access
import time
import sys

def make_vector_importer(db, stop_words_file, all_tags = {}, words = {}):
    db_access.read_stat_info_from_db(db, all_tags, words)
    stop_words = [line[:-1] for line in open(stop_words_file)]
    vectorizor = Vectorizor(all_tags, words, stop_words)

    def _import_vector(post_type, post):
        if post_type == "question":
            vector, tags = vectorizor.vectorize(post["Body"],
                                               post["Tags"])
            vector_doc = \
            {
                "id": post["Id"],
                "posts": [vector.items()],
                "tags": tags
            }
            db.vectors_table.insert(vector_doc)
        else:
            vector, tags = vectorizor.vectorize(post["Body"], {})
            parent_id = post["ParentId"]
            db.vectors_table.update(
                     {"id": parent_id},
                     {"$push": {"posts": vector.items()}})

    return _import_vector

def make_post_importer(db):
    def _import_post(post_type, post):
        if post_type == "question":
            db.question_table.insert(post)
        elif post_type == "answer":<commit_msg>Also vectorize the title of the post<commit_after>from vectorizor import Vectorizor
import db_access
import time
import sys

def make_vector_importer(db, stop_words_file, all_tags = {}, words = {}):
    db_access.read_stat_info_from_db(db, all_tags, words)
    stop_words = [line[:-1] for line in open(stop_words_file)]
    vectorizor = Vectorizor(all_tags, words, stop_words)

    def _import_vector(post_type, post):
        if post_type == "question":
            content = "%s %s" % (post["Body"], post["Title"])
            vector, tags = vectorizor.vectorize(content,
                                               post["Tags"])
            vector_doc = \
            {
                "id": post["Id"],
                "posts": [vector.items()],
                "tags": tags
            }
            db.vectors_table.insert(vector_doc)
        else:
            vector, tags = vectorizor.vectorize(post["Body"], {})
            parent_id = post["ParentId"]
            db.vectors_table.update(
                     {"id": parent_id},
                     {"$push": {"posts": vector.items()}})

    return _import_vector

def make_post_importer(db):
    def _import_post(post_type, post):
        if post_type == "question":
            db.question_table.insert(post)
        elif post_type == "answer":
{'fraction_non_alphanumeric': 0.12165354330708661, 'fraction_numerical': 0.0007874015748031496, 'mean_word_length': 2.032219570405728, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6583297', 'n_tokens_mistral': 746, 'n_tokens_neox': 728, 'n_words': 154}