<commit_before>import pymongo
from pymongo import MongoClient
from collections import defaultdict

class Searcher:
  def __init__ (self):
    self.client = MongoClient('10.131.137.188', 27017)
    self.db = self.client.grupo_04
    self.db.authenticate("user1", "sepelu.2017")
    self.goofle = self.db.goofle
    self.vowels = {"á":"a", "à":"a", "â":"a", "ä":"a", "é":"e",
                   "è":"e", "ê":"e", "ë":"e", "í":"i", "ì":"i",
                   "î":"i", "ï":"i", "ó":"o", "ò":"o", "ô":"o",
                   "ö":"o", "ú":"u", "ù":"u", "û":"u", "ü":"u",
                   "ñ":"n"}
  
  def search (self, words, top):
    result = defaultdict(lambda:0)
    for word in words:
      word = word.lower()
      for v, c in self.vowels.items():
        word = word.replace(v, c)
      files = self.goofle.find_one({"word": word}, {"_id": False, "files": True})
      if files is None:
        continue
      for _file in files["files"]:
        file, count = _file.values()
        result[file] += count
    resultlist = sorted(result.items(), key=lambda x:x[1], reverse = True)
    return resultlist[:top]
<commit_msg>Added all the special chars to the processing script for cleaning<commit_after>﻿import pymongo
from pymongo import MongoClient
from collections import defaultdict

class Searcher:
  def __init__ (self):
    self.client = MongoClient('10.131.137.188', 27017)
    self.db = self.client.grupo_04
    self.db.authenticate("user1", "sepelu.2017")
    self.goofle = self.db.goofle
    self.spchars = {"á":"a", "à":"a", "â":"a", "ä":"a", "é":"e",
                   "è":"e", "ê":"e", "ë":"e", "í":"i", "ì":"i",
                   "î":"i", "ï":"i", "ó":"o", "ò":"o", "ô":"o",
                   "ö":"o", "ú":"u", "ù":"u", "û":"u", "ü":"u",
                   "ñ":"n", "!":"", "|":"", "°":"", "¬":"", "\"":""
                   "#":"", "$":"", "%":"", "&":"", "/":"", "(":"", 
                   ")":"", "=":"", "'":"", "?":"", "¡":"", "¿":""
                   "@":"", "¨":"", "´":"", "+":"", "*":"", "~":""
                   "<":"", ">":"", "{":"", "[":"", "^":"", "}":""
                   "`":"", "]":"", ";":"", ",":"", ".":"", ":":""
                   "-":"", "_":"" }
  
  def search (self, words, top):
    result = defaultdict(lambda:0)
    for word in words:
      word = word.lower()
      for v, c in self.spchars.items():
        word = word.replace(v, c)
      files = self.goofle.find_one({"word": word}, {"_id": False, "files": True})
      if files is None:
        continue
      for _file in files["files"]:
        file, count = _file.values()
        result[file] += count
    resultlist = sorted(result.items(), key=lambda x:x[1], reverse = True)
    return resultlist[:top]

{'fraction_non_alphanumeric': 0.27051895472948106, 'fraction_numerical': 0.01840264998159735, 'mean_word_length': 2.4318181818181817, 'pattern_counts': {'":': 86, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4354848', 'n_tokens_mistral': 1004, 'n_tokens_neox': 921, 'n_words': 208}