<commit_before>import dill
import multiprocessing as mp
import pandas as pd


def apply_worker(result_queue, data, offset, chunk_size, f_pickled, kwargs):
    # Unpickle the function
    f = dill.loads(f_pickled)
    # Apply function and store result
    chunk = data[offset*chunk_size:(offset+1)*chunk_size]
    result_queue.put((offset, chunk.apply(f, **kwargs)))
    return True
<commit_msg>added set_start_method('spawn')<commit_after>import multiprocessing as mp
mp.set_start_method('spawn')
import pandas as pd
import dill

def apply_worker(result_queue, data, offset, chunk_size, f_pickled, kwargs):
    # Unpickle the function
    f = dill.loads(f_pickled)
    # Apply function and store result
    chunk = data[offset*chunk_size:(offset+1)*chunk_size]
    result_queue.put((offset, chunk.apply(f, **kwargs)))
    return True

{'fraction_non_alphanumeric': 0.10191846522781775, 'fraction_numerical': 0.002398081534772182, 'mean_word_length': 5.094890510948905, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '741102', 'n_tokens_mistral': 275, 'n_tokens_neox': 273, 'n_words': 70}