<commit_before>#!/usr/bin/env python
# -*- coding: utf-8

from sys import stdout
from math import log
from codecs import open
from argparse import ArgumentParser

def main():

    parser = ArgumentParser()
    parser.add_argument("seg_file", help = "segment file with term weight one query/bidword per line in tsv format")
    args = parser.parse_args()
    
    seg_file = args.seg_file

    word_dict = {}
    phrase_counter = 0.0
    with open(seg_file, "r", encoding = "utf-8") as fd:
        for line in fd:
            splited_line = line.strip().split("\t")
            if len(splited_line) != 2:
                continue
            phrase_counter += 1
            phrase_str, phrase_seg = splited_line
            for token in phrase_seg.split(""):
                word = token.split("")[0]
                if word in word_dict:
                    word_dict[word] += 1
                else:
                    word_dict[word] = 1
    for word in word_dict:
        word_dict[word] = log(phrase_counter / word_dict[word])

    with open(seg_file, "r", encoding = "utf-8") as fd:
        for line in fd:
            splited_line = line.strip().split("\t")
            if len(splited_line) != 2:
                continue
            phrase_str, phrase_seg = splited_line
            word_list = [(token.split("")[0], float(token.split("")[1])) for token in phrase_seg.split("")]
            normalize = sum([word_dict[word[0]] for word in word_list])
            res_list = []
            for word in word_list:
                res_list.append("%s/%f/%f" % (word[0], word[1], word_dict[word[0]] / normalize))
            print "%s\t%s" % (phrase_str.encode("utf-8"), " ".join(res_list).encode("utf-8"))

if __name__ == "__main__":

    main()

<commit_msg>Modify global-weight.py, add context-free alg for weighting.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8

from sys import stdout
from math import log
from codecs import open
from argparse import ArgumentParser
from numpy import mean

def main():

    parser = ArgumentParser()
    parser.add_argument("query_file", help = "query segment file (original query and segmented query) one query per line in tsv format")
    parser.add_argument("bidword_file", help = "bidword segment file (original bidword and segmented bidword) one query per line in tsv format")
    parser.add_argument("stopword_file", help = "one stopword per line in tsv format")
    args = parser.parse_args()
    
    query_file = args.query_file
    bidword_file = args.bidword_file
    stopword_file = args.stopword_file

    iter_count = 50

    word_dict = {}

    # idf
    phrase_counter = 0.0
    with open(query_file, "r", encoding = "utf-8") as fd:
        for line in fd:
            splited_line = line.strip().split("\t")
            if len(splited_line) != 2:
                continue
            phrase_counter += 1
            phrase_str, phrase_seg = splited_line
            for word in phrase_seg.split():
                if word in word_dict:
                    word_dict[word] += 1
                else:
                    word_dict[word] = 1
    with open(bidword_file, "r", encoding = "utf-8") as fd:
        for line in fd:
            splited_line = line.strip().split("\t")
            if len(splited_line) != 2:
                continue
            phrase_counter += 1
            phrase_str, phrase_seg = splited_line
            for word in phrase_seg.split():
                if word in word_dict:
                    word_dict[word] += 1
                else:
                    word_dict[word] = 1
    for word in word_dict:
        word_dict[word] = log(phrase_counter / word_dict[word])

    # context-free weight
    assist_dict = dict((word, []) for word in word_dict)
    for i in xrange(iter_count):
        with open(query_file, "r", encoding = "utf-8") as fd:
            for line in fd:
                splited_line = line.strip().split("\t")
                if len(splited_line) != 2:
                    continue
                phrase_str, phrase_seg = splited_line
                word_set = set(phrase_seg.split())
                weight_sum = sum([word_dict[word] for word in word_set if word in word_dict])
                for word in word_set:
                    if word in assist_dict:
                        assist_dict[word].append(word_dict[word] / weight_sum)
        with open(bidword_file, "r", encoding = "utf-8") as fd:
            for line in fd:
                splited_line = line.strip().split("\t")
                if len(splited_line) != 2:
                    continue
                phrase_str, phrase_seg = splited_line
                word_set = set(phrase_seg.split())
                weight_sum = sum([word_dict[word] for word in word_set if word in word_dict])
                for word in word_set:
                    if word in assist_dict:
                        assist_dict[word].append(word_dict[word] / weight_sum)
        for word in word_dict:
            if word in assist_dict:
                word_dict[word] = mean(assist_dict[word])
            assist_dict[word] = []
                    
    # remove stopwords
    with open(stopword_file, "r", encoding = "utf-8") as fd:
        for line in fd:
            splited_line = line.strip().split("\t")
            if len(splited_line) != 1:
                continue
            stopword = splited_line[0]
            if stopword in word_dict:
                word_dict[stopword] = 0.0

    for word in word_dict:
        print "%s\t%f" % (word.encode("utf-8"), word_dict[word])


if __name__ == "__main__":

    main()


{'fraction_non_alphanumeric': 0.10651204281891169, 'fraction_numerical': 0.00785013380909902, 'mean_word_length': 1.8719262295081966, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5169009', 'n_tokens_mistral': 1706, 'n_tokens_neox': 1631, 'n_words': 454}