<commit_before>#!/usr/bin/env python3

"""
This module provides functions in order to count

* words and symbols in a corpus file
* cues and outcomes in an event file

"""

import sys
import os
from collections import Counter
import itertools
import functools
import multiprocessing

import pickle


def _job_cues_outcomes(event_file_name, start, step, verbose=True):
    """
    Counts cues and outcomes for every ``step`` event starting from
    ``start`` event.

    Returns
    =======
    (cues, outcomes) : (collections.Counter, collections.Counter)

    """
    cues = Counter()
    outcomes = Counter()
    with open(event_file_name, 'r') as dfile:
        # skip header
        dfile.readline()
        for nn, line in enumerate(itertools.islice(dfile, start, None, step)):
            cues_line, outcomes_line, freq = line.split('\t')
            freq = int(freq)
            for cue in cues_line.split('_'):
                cues[cue] += freq
            for outcome in outcomes_line.strip().split('_'):
                outcomes[outcome] += freq
            if verbose and nn % 100000 == 0:
                print('.', end='')
                sys.stdout.flush()
    return (cues, outcomes)


def cues_outcomes(event_file_name,
                  *, number_of_processes=2, verbose=True):
    """
    Counts cues and outcomes in event_file_name using number_of_processes
    processes.

    Returns
    =======
    (cues, outcomes) : (collections.Counter, collections.Counter)

    """
    with multiprocessing.Pool(number_of_processes) as pool:
        step = num_of_processes
        results = pool.starmap(_job_cues_outcomes, ((event_file_name,
                                                    start,
                                                    step,
                                                    verbose)
                                      for start in range(number_of_processes)))
        cues = Counter()
        outcomes = Counter()
        for cues_process, outcomes_process in results:
            cues += cues_process
            outcomes += outcomes_process

    if verbose:
        print('\n...counting done.')

    return cues, outcomes<commit_msg>small fix in counting.py<commit_after>#!/usr/bin/env python3

"""
This module provides functions in order to count

* words and symbols in a corpus file
* cues and outcomes in an event file

"""

import sys
import os
from collections import Counter
import itertools
import functools
import multiprocessing

import pickle


def _job_cues_outcomes(event_file_name, start, step, verbose=True):
    """
    Counts cues and outcomes for every ``step`` event starting from
    ``start`` event.

    Returns
    =======
    (cues, outcomes) : (collections.Counter, collections.Counter)

    """
    cues = Counter()
    outcomes = Counter()
    with open(event_file_name, 'r') as dfile:
        # skip header
        dfile.readline()
        for nn, line in enumerate(itertools.islice(dfile, start, None, step)):
            cues_line, outcomes_line, freq = line.split('\t')
            freq = int(freq)
            for cue in cues_line.split('_'):
                cues[cue] += freq
            for outcome in outcomes_line.strip().split('_'):
                outcomes[outcome] += freq
            if verbose and nn % 100000 == 0:
                print('.', end='')
                sys.stdout.flush()
    return (cues, outcomes)


def cues_outcomes(event_file_name,
                  *, number_of_processes=2, verbose=True):
    """
    Counts cues and outcomes in event_file_name using number_of_processes
    processes.

    Returns
    =======
    (cues, outcomes) : (collections.Counter, collections.Counter)

    """
    with multiprocessing.Pool(number_of_processes) as pool:
        step = number_of_processes
        results = pool.starmap(_job_cues_outcomes, ((event_file_name,
                                                    start,
                                                    step,
                                                    verbose)
                                      for start in range(number_of_processes)))
        cues = Counter()
        outcomes = Counter()
        for cues_process, outcomes_process in results:
            cues += cues_process
            outcomes += outcomes_process

    if verbose:
        print('\n...counting done.')

    return cues, outcomes
{'fraction_non_alphanumeric': 0.0954876937101185, 'fraction_numerical': 0.004102096627164996, 'mean_word_length': 1.916279069767442, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3890168', 'n_tokens_mistral': 1242, 'n_tokens_neox': 1161, 'n_words': 334}