<commit_before># Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
import frappe

from whoosh.index import create_in, open_dir, EmptyIndexError
from whoosh.fields import TEXT, ID, Schema
from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin
from whoosh.query import Prefix

class FullTextSearch:
	""" Frappe Wrapper for Whoosh """

	def __init__(self, index_name):
		self.index_name = index_name
		self.index_path = get_index_path(index_name)
		self.schema = self.get_schema()
		self.id = self.get_id()

	def get_schema(self):
		return Schema(name=ID(stored=True), content=TEXT(stored=True))

	def get_id(self):
		return "name"

	def get_items_to_index(self):
		"""Get all documents to be indexed conforming to the schema"""
		return []

	def get_document_to_index(self):
		return {}

	def build(self):
		"""	Build search index for all documents """
		self.documents = self.get_items_to_index()
		self.build_index()

	def update_index_by_name(self, doc_name):
		"""Wraps `update_index` method, gets the document from name
		and updates the index. This function changes the current user
		and should only be run as administrator or in a background job.

		Args:
			self (object): FullTextSearch Instance
			doc_name (str): name of the document to be updated
		"""
		document = self.get_document_to_index(doc_name)
		self.update_index(document)

	def remove_document_from_index(self, doc_name):
		"""Remove document from search index

		Args:
			self (object): FullTextSearch Instance
			doc_name (str): name of the document to be removed
		"""
		ix = self.get_index()
		with ix.searcher():
			writer = ix.writer()
			writer.delete_by_term(self.id, doc_name)
			writer.commit(optimize=True)

	def update_index(self, document):
		"""Update search index for a document

		Args:
			self (object): FullTextSearch Instance
			document (_dict): A dictionary with title, path and content
		"""<commit_msg>feat: delete doc only if valid name is provided<commit_after># Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
import frappe

from whoosh.index import create_in, open_dir, EmptyIndexError
from whoosh.fields import TEXT, ID, Schema
from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin
from whoosh.query import Prefix

class FullTextSearch:
	""" Frappe Wrapper for Whoosh """

	def __init__(self, index_name):
		self.index_name = index_name
		self.index_path = get_index_path(index_name)
		self.schema = self.get_schema()
		self.id = self.get_id()

	def get_schema(self):
		return Schema(name=ID(stored=True), content=TEXT(stored=True))

	def get_id(self):
		return "name"

	def get_items_to_index(self):
		"""Get all documents to be indexed conforming to the schema"""
		return []

	def get_document_to_index(self):
		return {}

	def build(self):
		"""	Build search index for all documents """
		self.documents = self.get_items_to_index()
		self.build_index()

	def update_index_by_name(self, doc_name):
		"""Wraps `update_index` method, gets the document from name
		and updates the index. This function changes the current user
		and should only be run as administrator or in a background job.

		Args:
			self (object): FullTextSearch Instance
			doc_name (str): name of the document to be updated
		"""
		document = self.get_document_to_index(doc_name)
		self.update_index(document)

	def remove_document_from_index(self, doc_name):
		"""Remove document from search index

		Args:
			self (object): FullTextSearch Instance
			doc_name (str): name of the document to be removed
		"""
		if not doc_name:
			return

		ix = self.get_index()
		with ix.searcher():
			writer = ix.writer()
			writer.delete_by_term(self.id, doc_name)
			writer.commit(optimize=True)

	def update_index(self, document):
		"""Update search index for a document

		Args:
			self (object): FullTextSearch Instance
			document (_dict): A dictionary with title, path and content
		"""
{'fraction_non_alphanumeric': 0.09180166912125676, 'fraction_numerical': 0.0019636720667648502, 'mean_word_length': 4.683403068340307, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3290006', 'n_tokens_mistral': 1399, 'n_tokens_neox': 1266, 'n_words': 410}