<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import logging
import tornado.web
import tornado.escape as esc
import engine.config
from engine.utils import construct_regex
from engine.parser import SearchParser
from engine.filters import http_checker
from engine.filters import gather_words_around_search_word as gwasw


class SearchHandler(tornado.web.RequestHandler):
    def initialize(self, database, parser):
        self.database = database
        self.parser = parser


    def get(self):
        # Show an empty webpage ready to search
        self.render("main.html", results=[])


    def bold_(self, word):
        return "{0}{1}{2}".format("<strong>",esc.xhtml_escape(word),"</strong>")


    def escape_and_bold(self, data, search_string):
        for word in search_string:
            data = data.replace(word, self.bold_(word))
        return data


    def post(self):
        # TODO: Clean up search from useless spaces
        search_string = self.parser.parse_input(self.get_argument('search_string').lower())
        search_string = re.split(re.compile(r'\s+') , search_string)

        matched_results = [
            {
                # Get the main part of the crawled webpage
                # TODO: make gwasw configurable
                'content': self.escape_and_bold(
                    gwasw(match['data'], search_string, 25),
                    search_string
                ),
                # Get the url
                'url': http_checker(match['url'])
            } for match in self.database.get_lists_collection().find({
                "data": { '$regex': construct_regex(search_string)}
            }) # Search mongodb<commit_msg>Escaping data before bold-ing search_string words<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import logging
import tornado.web
import tornado.escape as esc
import engine.config
from engine.utils import construct_regex
from engine.parser import SearchParser
from engine.filters import http_checker
from engine.filters import gather_words_around_search_word as gwasw


class SearchHandler(tornado.web.RequestHandler):
    def initialize(self, database, parser):
        self.database = database
        self.parser = parser


    def get(self):
        # Show an empty webpage ready to search
        self.render("main.html", results=[])


    def bold_(self, word):
        return "{0}{1}{2}".format("<strong>",esc.xhtml_escape(word),"</strong>")


    def escape_and_bold(self, data, search_string):
        data = esc.xhtml_escape(data)
        for word in search_string:
            data = data.replace(word, self.bold_(word))
        return data


    def post(self):
        # TODO: Clean up search from useless spaces
        search_string = self.parser.parse_input(self.get_argument('search_string').lower())
        search_string = re.split(re.compile(r'\s+') , search_string)

        matched_results = [
            {
                # Get the main part of the crawled webpage
                # TODO: make gwasw configurable
                'content': self.escape_and_bold(
                    gwasw(match['data'], search_string, 25),
                    search_string
                ),
                # Get the url
                'url': http_checker(match['url'])
            } for match in self.database.get_lists_collection().find({
                "data": { '$regex': construct_regex(search_string)}
            }) # Search mongodb
{'fraction_non_alphanumeric': 0.10455069124423963, 'fraction_numerical': 0.0034562211981566822, 'mean_word_length': 2.4284304047384007, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4779493', 'n_tokens_mistral': 973, 'n_tokens_neox': 912, 'n_words': 246}