<commit_before>#!/usr/bin/env python
###############################################################################
 #  Copyright (c) 2015 <NAME>
 #  Project: 
 #  Filename: traverse.py 
 #  Version: 1.0
 #  Author: <NAME>
 #  E-mail: <EMAIL>
 #  Date: Aug  3, 2015
 #  Time: 14:06:02
 #  Description: 
###############################################################################

from __future__ import print_function
import sys
import os
import multiprocessing
import time
import Queue

import config_load
import traverse_dir 
import traverse_urllist 


# command line arguments: lib_path conf_path log path
if __name__ == "__main__":
    if len(sys.argv) < 4:
        print("Error: Command line arguments error. Please view source code to get help. ")
        exit(1)

    lib_path = str(sys.argv[1])
    conf_path = str(sys.argv[2])
    log_path = str(sys.argv[3])

    if not os.path.isabs(conf_path):
        print("Error: Configuration path", conf_path, "is not absolute path. ")
        exit(1)

    if not os.path.isabs(lib_path):
        print("Error: Library pth", lib_path, "is not absolute path. ")
        exit(1)

    if not os.path.isabs(log_path):
        print("Error: Log path", log_path, "is not absolute path. ")
        exit(1)

    if not os.path.exists(log_path):
        os.mkdir(log_path)

    if not os.path.isdir(log_path):
        print("Error: Log path", log_path, "is not directory or not exists. ")
        exit(1)
    
    log_files = [ "stdout", "stderr", ]
    log_files = [ os.path.join(log_path, x) for x in log_files ] 

    for log_file in log_files:
        if os.path.exists(log_file):
            print("Error:", log_file, "alreadty exists. ")
            exit(1)



    # non-builtin modules mustn't be loaded before this statement
    sys.path.insert(0, lib_path)
    from job_queue import JobQueue

    # load config
    config = config_load.load_config(conf_path)

    # print(config)

    migrate_type = ("migrateinfo", "migrate.type")

    if migrate_type[0] not in config or migrate_type[1] not in config[migrate_type[0]]:
        print("Error: Option", migrate_type[0] + "." + migrate_type[1], "is required. ")

    traverse_functions = [ 
                           traverse_dir.traverse,
                           traverse_urllist.traverse,
                           # other traverse methods
                         ]

    #traverse_functions[int(config[migrate_type[0]][migrate_type[1]]) - 1](config, log_path)


    # start parallel uploading
    try:
        message_queue = multiprocessing.Queue()
        
        num_processes = int(config["toolconfig"]["concurrency"])
        job_queue = JobQueue(
                             num_processes,
                             config["appinfo"]["appinfo.appid"],<commit_msg>log pid<commit_after>#!/usr/bin/env python
###############################################################################
 #  Copyright (c) 2015 <NAME>
 #  Project: 
 #  Filename: traverse.py 
 #  Version: 1.0
 #  Author: <NAME>
 #  E-mail: <EMAIL>
 #  Date: Aug  3, 2015
 #  Time: 14:06:02
 #  Description: 
###############################################################################

from __future__ import print_function
import sys
import os
import multiprocessing
import time
import Queue

import config_load
import traverse_dir 
import traverse_urllist 


# command line arguments: lib_path conf_path log path
if __name__ == "__main__":
    if len(sys.argv) < 4:
        print("Error: Command line arguments error. Please view source code to get help. ")
        exit(1)

    lib_path = str(sys.argv[1])
    conf_path = str(sys.argv[2])
    log_path = str(sys.argv[3])

    if not os.path.isabs(conf_path):
        print("Error: Configuration path", conf_path, "is not absolute path. ")
        exit(1)

    if not os.path.isabs(lib_path):
        print("Error: Library pth", lib_path, "is not absolute path. ")
        exit(1)

    if not os.path.isabs(log_path):
        print("Error: Log path", log_path, "is not absolute path. ")
        exit(1)

    if not os.path.exists(log_path):
        os.mkdir(log_path)

    if not os.path.isdir(log_path):
        print("Error: Log path", log_path, "is not directory or not exists. ")
        exit(1)
    
    log_files = [ "stdout", "stderr", "pid" ]
    log_files = [ os.path.join(log_path, x) for x in log_files ] 

    for log_file in log_files:
        if os.path.exists(log_file):
            print("Error:", log_file, "alreadty exists. ")
            exit(1)



    # non-builtin modules mustn't be loaded before this statement
    sys.path.insert(0, lib_path)
    from job_queue import JobQueue

    # load config
    config = config_load.load_config(conf_path)

    # print(config)

    migrate_type = ("migrateinfo", "migrate.type")

    if migrate_type[0] not in config or migrate_type[1] not in config[migrate_type[0]]:
        print("Error: Option", migrate_type[0] + "." + migrate_type[1], "is required. ")
        exit(1)

    traverse_functions = [ 
                           traverse_dir.traverse,
                           traverse_urllist.traverse,
                           # other traverse methods
                         ]

    with open(os.path.join(log_path, "pid"), "w") as f:
        f.write(str(os.getpid()))

    # start parallel uploading
    try:
        message_queue = multiprocessing.Queue()
        
        num_processes = int(config["toolconfig"]["concurrency"])
        job_queue = JobQueue(
                             num_processes,
                             config["appinfo"]["appinfo.appid"],
{'fraction_non_alphanumeric': 0.16445718380508778, 'fraction_numerical': 0.012540308133285561, 'mean_word_length': 2.5583173996175907, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7180892', 'n_tokens_mistral': 1790, 'n_tokens_neox': 1698, 'n_words': 422}