<commit_before>import math
import numpy as np
import json
import operator
import re

from tsg.config import DICTIONARY_PATH, INDEXINFO_PATH
from tsg.ranker.hasher import hash_index_terms


def get_dictionary_term_list(term,index_dictionary_path=DICTIONARY_PATH):

    if not get_dictionary_term_list.index_hash:
        get_dictionary_term_list.index_hash = hash_index_terms(index_dictionary_path)

    document_list = {}
    with open(index_dictionary_path) as dict_f:
        dict_f.seek(get_dictionary_term_list.index_hash[term][0])
        line_term, documents = dict_f.readline().replace('\n', '').split(' ')

        assert term == line_term

        for uuid, weight in re.findall('(?:,|^)(.*?):([0-9]+\.[0-9]*)',
                                      documents):
            document_list[uuid] = float(weight)

    return document_list
get_dictionary_term_list.index_hash = None

def get_number_of_docs(index_dictionary_path):
    dict_f = { }
    with open(index_dictionary_path) as info:
        dict_f = json.load(info)

    return dict_f["num_documents"]


def calculate_query_term_weight(term, query_terms, index_dictionary_path=DICTIONARY_PATH,
    index_info_path = INDEXINFO_PATH):
    term_dictionary = get_dictionary_term_list(term, index_dictionary_path)
    N = get_number_of_docs(index_info_path)
    doc_freq = len(term_dictionary)
    term_freq = query_terms.count(term)
    try:
        weight = (1+ np.log10(term_freq))*math.log10(N/doc_freq)
    except ZeroDivisionError:
        weight = 0

    return weight

def and_score_calc(query_terms, index_dictionary_path= DICTIONARY_PATH,
    index_info_path = INDEXINFO_PATH):

    common__doc_keys = set()
    terms_documents = {}
    and_scored_docs = {}
    doc_length = {}
    query_length = {}

    for term in query_terms:
        terms_documents[term] = get_dictionary_term_list(term, index_dictionary_path)
        if len(common__doc_keys) == 0:
            common__doc_keys = terms_documents[term].keys()
        else:
            common__doc_keys &= terms_documents[term].keys()

    for key in common__doc_keys:
        for term in query_terms:
            if key in terms_documents[term].keys():
                query_term_weight = calculate_query_term_weight(term,query_terms,
            index_dictionary_path, index_info_path)
                if key in and_scored_docs:
                    and_scored_docs[key] += float(terms_documents[term][key])*float(query_term_weight)
                else:
                    and_scored_docs[key] = float(terms_documents[term][key])*float(query_term_weight)

                if key in doc_length:
                    doc_length[key] += math.pow(float(terms_documents[term][key]), float(2))
                    query_length[key] += math.pow(float(query_term_weight), float(2))
                else:
                    doc_length[key] = math.pow(float(terms_documents[term][key]), float(2))
                    query_length[key] = math.pow(float(query_term_weight), float(2))

    for key in and_scored_docs:
        try:
            and_scored_docs[key] = and_scored_docs[key] / (math.sqrt(doc_length[key])*math.sqrt(query_length[key]))
        except ZeroDivisionError:
            and_scored_docs[key] = 0

    return and_scored_docs

def or_score_calc(query_terms, index_dictionary_path=DICTIONARY_PATH,
    index_info_path = INDEXINFO_PATH):

    or_scored_docs = {}
    query_length = {} # Holds score^2 for Length normalization at end
    doc_length = {}
    for term in query_terms:<commit_msg>performance tweak<commit_after>import math
import numpy as np
import json
import operator
import re

from tsg.config import DICTIONARY_PATH, INDEXINFO_PATH
from tsg.ranker.hasher import hash_index_terms


def get_dictionary_term_list(term,index_dictionary_path=DICTIONARY_PATH):

    if not get_dictionary_term_list.index_hash:
        get_dictionary_term_list.index_hash = hash_index_terms(index_dictionary_path)

    document_list = {}
    with open(index_dictionary_path) as dict_f:
        dict_f.seek(get_dictionary_term_list.index_hash[term][0])
        line_term, documents = dict_f.readline().replace('\n', '').split(' ')

        assert term == line_term

        for uuid, weight in re.findall('(?:,|^)(.*?):([0-9]+\.[0-9]*)',
                                      documents):
            document_list[uuid] = float(weight)

    return document_list
get_dictionary_term_list.index_hash = None

def get_number_of_docs(index_dictionary_path):
    dict_f = { }
    with open(index_dictionary_path) as info:
        dict_f = json.load(info)

    return dict_f["num_documents"]


def calculate_query_term_weight(term, query_terms, index_dictionary_path=DICTIONARY_PATH,
    index_info_path = INDEXINFO_PATH):
    term_dictionary = get_dictionary_term_list(term, index_dictionary_path)
    N = get_number_of_docs(index_info_path)
    doc_freq = len(term_dictionary)
    term_freq = query_terms.count(term)
    try:
        weight = (1+ np.log10(term_freq))*math.log10(N/doc_freq)
    except ZeroDivisionError:
        weight = 0

    return weight

def and_score_calc(query_terms, index_dictionary_path= DICTIONARY_PATH,
    index_info_path = INDEXINFO_PATH):

    common__doc_keys = set()
    terms_documents = {}
    and_scored_docs = {}
    doc_length = {}
    query_length = {}

    for term in query_terms:
        terms_documents[term] = get_dictionary_term_list(term, index_dictionary_path)
        if len(common__doc_keys) == 0:
            common__doc_keys = terms_documents[term].keys()
        else:
            common__doc_keys &= terms_documents[term].keys()

    for key in common__doc_keys:
        for term in query_terms:
            if key in terms_documents[term]:
                query_term_weight = calculate_query_term_weight(term,query_terms,
            index_dictionary_path, index_info_path)
                if key in and_scored_docs:
                    and_scored_docs[key] += float(terms_documents[term][key])*float(query_term_weight)
                else:
                    and_scored_docs[key] = float(terms_documents[term][key])*float(query_term_weight)

                if key in doc_length:
                    doc_length[key] += math.pow(float(terms_documents[term][key]), float(2))
                    query_length[key] += math.pow(float(query_term_weight), float(2))
                else:
                    doc_length[key] = math.pow(float(terms_documents[term][key]), float(2))
                    query_length[key] = math.pow(float(query_term_weight), float(2))

    for key in and_scored_docs:
        try:
            and_scored_docs[key] = and_scored_docs[key] / (math.sqrt(doc_length[key])*math.sqrt(query_length[key]))
        except ZeroDivisionError:
            and_scored_docs[key] = 0

    return and_scored_docs

def or_score_calc(query_terms, index_dictionary_path=DICTIONARY_PATH,
    index_info_path = INDEXINFO_PATH):

    or_scored_docs = {}
    query_length = {} # Holds score^2 for Length normalization at end
    doc_length = {}
    for term in query_terms:
{'fraction_non_alphanumeric': 0.09488015884271735, 'fraction_numerical': 0.005105658771805418, 'mean_word_length': 3.1457965902410345, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1831270', 'n_tokens_mistral': 2350, 'n_tokens_neox': 2193, 'n_words': 382}