<commit_before># Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

import os

from bs4 import BeautifulSoup
from whoosh.fields import ID, TEXT, Schema

import frappe
from frappe.search.full_text_search import FullTextSearch
from frappe.utils import set_request, update_progress_bar
from frappe.website.render import render_page

INDEX_NAME = "web_routes"

class WebsiteSearch(FullTextSearch):
	""" Wrapper for WebsiteSearch """

	def get_schema(self):
		return Schema(
			title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True)
		)

	def get_id(self):
		return "path"

	def get_items_to_index(self):
		"""Get all routes to be indexed, this includes the static pages
		in www/ and routes from published documents

		Returns:
			self (object): FullTextSearch Instance
		"""

		if getattr(self, "_items_to_index", False):
			return self._items_to_index

		routes = get_static_pages_from_all_apps() + slugs_with_web_view()

		self._items_to_index = []

		for i, route in enumerate(routes):
			update_progress_bar(f"Retrieving Routes", i, len(routes))
			self._items_to_index += [self.get_document_to_index(route)]

		print()

		return self.get_items_to_index()

	def get_document_to_index(self, route):
		"""Render a page and parse it using BeautifulSoup

		Args:
			path (str): route of the page to be parsed

		Returns:
			document (_dict): A dictionary with title, path and content
		"""
		frappe.set_user("Guest")
		frappe.local.no_cache = True

		try:
			set_request(method="GET", path=route)
			content = render_page(route)
			soup = BeautifulSoup(content, "html.parser")
			page_content = soup.find(class_="page_content")<commit_msg>chore: Remove f prefix on plain string<commit_after># Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

import os

from bs4 import BeautifulSoup
from whoosh.fields import ID, TEXT, Schema

import frappe
from frappe.search.full_text_search import FullTextSearch
from frappe.utils import set_request, update_progress_bar
from frappe.website.render import render_page

INDEX_NAME = "web_routes"

class WebsiteSearch(FullTextSearch):
	""" Wrapper for WebsiteSearch """

	def get_schema(self):
		return Schema(
			title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True)
		)

	def get_id(self):
		return "path"

	def get_items_to_index(self):
		"""Get all routes to be indexed, this includes the static pages
		in www/ and routes from published documents

		Returns:
			self (object): FullTextSearch Instance
		"""

		if getattr(self, "_items_to_index", False):
			return self._items_to_index

		routes = get_static_pages_from_all_apps() + slugs_with_web_view()

		self._items_to_index = []

		for i, route in enumerate(routes):
			update_progress_bar("Retrieving Routes", i, len(routes))
			self._items_to_index += [self.get_document_to_index(route)]

		print()

		return self.get_items_to_index()

	def get_document_to_index(self, route):
		"""Render a page and parse it using BeautifulSoup

		Args:
			path (str): route of the page to be parsed

		Returns:
			document (_dict): A dictionary with title, path and content
		"""
		frappe.set_user("Guest")
		frappe.local.no_cache = True

		try:
			set_request(method="GET", path=route)
			content = render_page(route)
			soup = BeautifulSoup(content, "html.parser")
			page_content = soup.find(class_="page_content")
{'fraction_non_alphanumeric': 0.09772462077012835, 'fraction_numerical': 0.0029171528588098016, 'mean_word_length': 4.942807625649913, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7534205', 'n_tokens_mistral': 1225, 'n_tokens_neox': 1114, 'n_words': 309}