<commit_before>import os

from whoosh.qparser import QueryParser
from whoosh.query import FuzzyTerm, And, Term

from whoosh import index
from whoosh.fields import Schema, ID, KEYWORD, TEXT

from whoosh.support.charset import accent_map
from whoosh.analysis import RegexTokenizer
from whoosh.analysis import CharsetFilter, LowercaseFilter, StopFilter

from lib.stopwords import stoplists

class IndexSchema():
    """
    Init schema and build a custom analyzer.

    All data to index will be put inside the
    """

    def __init__(self):

        chfilter = CharsetFilter(accent_map)
        stoplist = stoplists["en"].union(stoplists["fr"])
        analyzer = RegexTokenizer() | LowercaseFilter() | \
                   StopFilter(stoplist=stoplist) | chfilter

        self.schema = Schema(content=TEXT(analyzer=analyzer), 
                             docType=TEXT, 
                             docId=ID(stored=True), 
                             tags=KEYWORD)

        if not os.path.exists("indexes"):
            os.mkdir("indexes")
            self.index = index.create_in("indexes", self.schema)
        else:
            self.index = index.open_dir("indexes")



class Indexer():

    def index_doc(self, docType, doc, fields):
        """
        Add given doc to index, tag and given fields are stored.
        """

        indexSchema = IndexSchema()
        content = " ".join([doc[field] for field in fields])
        writer = indexSchema.index.writer()
        writer.update_document(content=unicode(content),
                                    docType=unicode(docType),
                                    docId=unicode(doc["id"]),
                                    tags=doc["tags"])
        writer.commit()


    def search_doc(self, word, docType):
        """
        Return a list of docs that contains given word and that matches
        given type.
        """

        indexSchema = IndexSchema()
        parser = QueryParser("content", schema=indexSchema.schema)
        query = parser.parse(word)
        query = And([query, Term("docType", unicode(docType.lower()))])

        with indexSchema.index.searcher() as searcher:
            results = searcher.search(query)
            return [result["docId"] for result in results]
        

    def remove_doc(self, id):
        """
        Remove given doc from index (doc of which docId is equal to id).
        """

        indexSchema = IndexSchema()

        writer = indexSchema.index.writer()
        writer.delete_by_term("docId", unicode(id))
        writer.commit()
<commit_msg>change search from standard to fuzzy<commit_after>import os

from whoosh.qparser import QueryParser
from whoosh.query import FuzzyTerm, And, Term

from whoosh import index
from whoosh.fields import Schema, ID, KEYWORD, TEXT

from whoosh.support.charset import accent_map
from whoosh.analysis import RegexTokenizer
from whoosh.analysis import CharsetFilter, LowercaseFilter, StopFilter

from lib.stopwords import stoplists

class IndexSchema():
    """
    Init schema and build a custom analyzer.

    All data to index will be put inside the
    """

    def __init__(self):

        chfilter = CharsetFilter(accent_map)
        stoplist = stoplists["en"].union(stoplists["fr"])
        analyzer = RegexTokenizer() | LowercaseFilter() | \
                   StopFilter(stoplist=stoplist) | chfilter

        self.schema = Schema(content=TEXT(analyzer=analyzer), 
                             docType=TEXT, 
                             docId=ID(stored=True), 
                             tags=KEYWORD)

        if not os.path.exists("indexes"):
            os.mkdir("indexes")
            self.index = index.create_in("indexes", self.schema)
        else:
            self.index = index.open_dir("indexes")



class Indexer():

    def index_doc(self, docType, doc, fields):
        """
        Add given doc to index, tag and given fields are stored.
        """

        indexSchema = IndexSchema()
        content = " ".join([doc[field] for field in fields])
        writer = indexSchema.index.writer()
        writer.update_document(content=unicode(content),
                                    docType=unicode(docType),
                                    docId=unicode(doc["id"]),
                                    tags=doc["tags"])
        writer.commit()


    def search_doc(self, word, docType):
        """
        Return a list of docs that contains given word and that matches
        given type.
        """

        indexSchema = IndexSchema()
        parser = QueryParser("content", schema=indexSchema.schema,
                termclass=FuzzyTerm)
        query = parser.parse(word)
        query = And([query, Term("docType", unicode(docType.lower()))])

        with indexSchema.index.searcher() as searcher:
            results = searcher.search(query)
            return [result["docId"] for result in results]
        

    def remove_doc(self, id):
        """
        Remove given doc from index (doc of which docId is equal to id).
        """

        indexSchema = IndexSchema()

        writer = indexSchema.index.writer()
        writer.delete_by_term("docId", unicode(id))
        writer.commit()

{'fraction_non_alphanumeric': 0.09888418622547133, 'fraction_numerical': 0.0, 'mean_word_length': 2.0928019036287924, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7007309', 'n_tokens_mistral': 1413, 'n_tokens_neox': 1356, 'n_words': 370}