<commit_before>import time
import json
import codecs
import sys
import multiprocessing as mp, os
import core
from optparse import OptionParser
""" Process code begins here """


class ParallelPtocess(object):
    """ Class to run the process in parallel """

    def __init__(self, input_path, output_path, config, processes=0):
        self.input = input_path
        self.output = output_path
        self.processes = processes
        self.core = core.Core(extraction_config=json.load(codecs.open(config, 'r')))

    @staticmethod
    def output_write(output_path):
        return codecs.open(output_path, 'w')

    @staticmethod
    def chunk_file(self, file_name, size=1024 * 1024):
        """ Splitting data into chunks for parallel processing
        :param file_name - name of the file to split
        :param size - size of file to split
        """
        file_end = os.path.getsize(file_name)
        with open(file_name, 'r') as f:
            chunk_end = f.tell()
            while True:
                chunk_start = chunk_end
                f.seek(size, 1)
                f.readline()
                chunk_end = f.tell()
                yield chunk_start, chunk_end - chunk_start
                if chunk_end > file_end:
                    break

    def process_wrapper(self, chunk_start, chunk_size):
        output = codecs.open(self.output, 'w')
        with open(self.input) as f:
            f.seek(chunk_start)
            lines = f.read(chunk_size).splitlines()<commit_msg>staticmethod should not have self<commit_after>import time
import json
import codecs
import sys
import multiprocessing as mp, os
import core
from optparse import OptionParser
""" Process code begins here """


class ParallelPtocess(object):
    """ Class to run the process in parallel """

    def __init__(self, input_path, output_path, config, processes=0):
        self.input = input_path
        self.output = output_path
        self.processes = processes
        self.core = core.Core(extraction_config=json.load(codecs.open(config, 'r')))

    @staticmethod
    def output_write(output_path):
        return codecs.open(output_path, 'w')

    @staticmethod
    def chunk_file(file_name, size=1024 * 1024):
        """ Splitting data into chunks for parallel processing
        :param file_name - name of the file to split
        :param size - size of file to split
        """
        file_end = os.path.getsize(file_name)
        with open(file_name, 'r') as f:
            chunk_end = f.tell()
            while True:
                chunk_start = chunk_end
                f.seek(size, 1)
                f.readline()
                chunk_end = f.tell()
                yield chunk_start, chunk_end - chunk_start
                if chunk_end > file_end:
                    break

    def process_wrapper(self, chunk_start, chunk_size):
        output = codecs.open(self.output, 'w')
        with open(self.input) as f:
            f.seek(chunk_start)
            lines = f.read(chunk_size).splitlines()
{'fraction_non_alphanumeric': 0.08913187541418158, 'fraction_numerical': 0.0066269052352551355, 'mean_word_length': 2.2254273504273505, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4961395', 'n_tokens_mistral': 894, 'n_tokens_neox': 847, 'n_words': 244}