<commit_before># Similarity and difference of multi thread vs. multi process
# Written by Vamei

import os
import threading
import multiprocessing
import queue

# worker function
def worker(sign, lock,qq):
    # lock.acquire()
    print(sign, os.getpid())
    print(__name__)
    # lock.release()



# # Multi-thread
# record = []
# lock  = threading.Lock()
# for i in range(5):
#     thread = threading.Thread(target=worker,args=('thread',lock))
#     thread.start()
#     record.append(thread)
#
# for thread in record:
#     thread.join()

# Multi-process
def test():
    record = []
    lock = multiprocessing.Lock()
    qq = multiprocessing.Queue()
    for i in range(5):
        process = multiprocessing.Process(target=worker,args=('process',lock,qq))
        record.append(process)

    for process in record:
        process.start()
        process.join()
    print(qq)

if __name__ == '__main__':
    print('hello world')
    # Main
    print('Main:', os.getpid())
    test()<commit_msg>adsfadf<commit_after># Similarity and difference of multi thread vs. multi process
# Written by Vamei

import os
import threading
import multiprocessing
import queue
import time

# worker function
def worker(sign, lock,qq):
    # lock.acquire()
    print(sign, os.getpid())
    print(__name__)
    time.sleep(5)
    # lock.release()



# # Multi-thread
# record = []
# lock  = threading.Lock()
# for i in range(5):
#     thread = threading.Thread(target=worker,args=('thread',lock))
#     thread.start()
#     record.append(thread)
#
# for thread in record:
#     thread.join()

# Multi-process
def test():
    record = []
    lock = multiprocessing.Lock()
    qq = multiprocessing.Queue()
    for i in range(5):
        process = multiprocessing.Process(target=worker,args=('process',lock,qq))
        process.start()
        record.append(process)

    for process in record:
        process.join()

def test2():
    pool = multiprocessing.Pool(5)
    for i in range(5):
        pool.apply_async(func=worker,args=('process2',None,None))
    pool.close()
    pool.join()


if __name__ == '__main__':
    print('hello world')
    # Main
    print('Main:', os.getpid())
    test()
    test2()
{'fraction_non_alphanumeric': 0.13939670932358317, 'fraction_numerical': 0.004570383912248629, 'mean_word_length': 3.18546845124283, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '885007', 'n_tokens_mistral': 721, 'n_tokens_neox': 709, 'n_words': 171}