<commit_before>import urllib
from collections import defaultdict
from itertools import chain

def merge_dicts(dict1, dict2):
    dict3 = defaultdict(list)
    for k, v in chain(dict1.items(), dict2.items()):
        for value in v:
            dict3[k].append(value)
    return dict3


def parse_query(query):
    return query.split()


def parse_query_string(parsed_query, queried_fields):
    query_string_dict = {}
    for queried_field in queried_fields:
        for query_item in parsed_query:
            query_string_dict = merge_dicts(query_string_dict,
                                            {queried_field:[query_item]})
    return urllib.parse.urlencode(query_string_dict, True)


def combine_redirect_url(redirect_url, query, queried_fields):
    parsed_query = parse_query(query)
    query_string = parse_query_string(parsed_query, queried_fields)
    redirect_parsed_url = urllib.parse.urlparse(redirect_url)
    redirect_parsed_url = redirect_parsed_url._replace(query=query_string)
    return urllib.parse.urlunparse(redirect_parsed_url)<commit_msg>Tokenize, cut the stop words and stem the query<commit_after>import urllib
from collections import defaultdict
from itertools import chain
import nltk
from nltk.corpus import stopwords


def merge_dicts(dict1, dict2):
    dict3 = defaultdict(list)
    for k, v in chain(dict1.items(), dict2.items()):
        for value in v:
            dict3[k].append(value)
    return dict3


def parse_query(query):
    tokenized_query = nltk.word_tokenize(query)
    lower_case_tokenized_query = [w.lower() for w in tokenized_query]
    tokenized_query_without_stopwords = [w for w in lower_case_tokenized_query if w not in stopwords.words('english')]
    print(tokenized_query_without_stopwords)
    stemmer = nltk.stem.PorterStemmer()
    stemmed_query = [stemmer.stem(word) for word in tokenized_query_without_stopwords]
    print(stemmed_query)
    return stemmed_query


def parse_query_string(parsed_query, queried_fields):
    query_string_dict = {}
    for queried_field in queried_fields:
        for query_item in parsed_query:
            query_string_dict = merge_dicts(query_string_dict,
                                            {queried_field:[query_item]})
    return urllib.parse.urlencode(query_string_dict, True)


def download_needed_packages_from_nltk():
    pass
    # nltk.download('punkt')
    # nltk.download('stopwords')


def combine_redirect_url(redirect_url, query, queried_fields):
    download_needed_packages_from_nltk()
    parsed_query = parse_query(query)
    query_string = parse_query_string(parsed_query, queried_fields)
    redirect_parsed_url = urllib.parse.urlparse(redirect_url)
    redirect_parsed_url = redirect_parsed_url._replace(query=query_string)
    return urllib.parse.urlunparse(redirect_parsed_url)
{'fraction_non_alphanumeric': 0.07577374599786553, 'fraction_numerical': 0.004980434009249377, 'mean_word_length': 4.057553956834532, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6301943', 'n_tokens_mistral': 895, 'n_tokens_neox': 893, 'n_words': 178}