<commit_before>from utils import configure_logger
import utils
import logging
import sys
from wiki.sc import create_page_id_link_mapping_file, get_categories_for_senses
import glob

logging.basicConfig(stream=sys.stdout, level=logging.INFO,
                    format='[%(asctime)s] p%(process)s %(name)s:%(lineno)d %(levelname)s - %(message)s')


if __name__ == '__main__':
    configure_logger("INFO")
    logger = utils.get_logger()
    import os
    files = glob.glob(os.path.abspath('../datasets/wiki') + "/*.tw.txt")
    create_page_id_link_mapping_file(files, '../datasets/wiki/')

    # get_categories_for_senses(['fiber.pageid.txt'], '/Users/obaskaya/Desktop/en_20090308/categorylink.csv',
    #                           '/Users/obaskaya/Desktop/en_20090306/page.csv',
    #                           '/Users/obaskaya/Desktop/en_20090306/generality.csv')<commit_msg>refactoring the code. adding argparse<commit_after>import os
import utils
import logging
import sys
import glob

from utils import configure_logger
from wiki.sc import create_page_id_link_mapping_file, get_categories_for_senses


logging.basicConfig(stream=sys.stdout, level=logging.INFO,
                    format='[%(asctime)s] p%(process)s %(name)s:%(lineno)d %(levelname)s - %(message)s')


def run():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--uwsd-dataset', required=False,
                        default='../datasets/wiki')
    parser.add_argument('--category-file', required=False,
                        default='../datasets/wikipedia-miner/en_20090306/categorylink.csv')
    parser.add_argument('--generality-file', required=False,
                        default='../datasets/wikipedia-miner/en_20090306/generality.csv')
    parser.add_argument('--pageid-title-file', required=False,
                        default='../datasets/wikipedia-miner/en_20090306/page.csv')
    parser.add_argument('--num-process', help="Number of process for parallel processing", required=False, default=1,
                        type=int)
    parser.add_argument('--log-level', required=False, default="info")
    args = parser.parse_args()

    configure_logger(args.log_level)
    logger = utils.get_logger()

    logger.info("Started")

    files = glob.glob(os.path.abspath(args.uwsd_dataset) + "/*.tw.txt")
    create_page_id_link_mapping_file(files, args.uwsd_dataset)
    files = glob.glob(os.path.abspath(args.uwsd_dataset) + "/*.pageid.txt")
    get_categories_for_senses(files, args.category_file, args.pageid_title_file, args.generality_file)

    logger.info("Done")


if __name__ == '__main__':
    run()
{'fraction_non_alphanumeric': 0.13568985176738882, 'fraction_numerical': 0.01862409730140631, 'mean_word_length': 3.4161073825503356, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5757708', 'n_tokens_mistral': 885, 'n_tokens_neox': 835, 'n_words': 124}