<commit_before>from __future__ import print_function
import csv
from itertools import takewhile
import json
import os
from itertools_recipes import grouper

LINE_BUFFER_SIZE = 5000


def ensure_dir(directory):
    """
    If directory doesn't exist, make it.

    :param directory: path to directory
    :type directory: `str`
    """
    if not os.path.exists(directory):
        os.makedirs(directory)


def i_get_csv_data(file_name, *args, **kwargs):
    """A generator for reading a csv file.
    """
    with open(file_name, 'r') as csv_file:
        reader = csv.reader(csv_file, *args, **kwargs)
        for row in reader:
            yield row


def write_as_csv(items, file_name, append=False, line_buffer_size=None):
    """
    Writes out tuples to a csv file
    """
    fill_object = object()
    if line_buffer_size is None:
        line_buffer_size = LINE_BUFFER_SIZE
    if append:
        mode = 'w+'
    else:
        mode = 'wt'
    with open(file_name, mode) as csv_file:
        writer = csv.writer(csv_file)
        line_groups = grouper(line_buffer_size,
                              items,
                              fillvalue=fill_object)

        for line_group in line_groups:
            writer.writerows(
                takewhile(lambda x: x is not fill_object, line_group))


def dump_dicts_to_json_file(file_name, dicts):
    """writes each dictionary in the dicts iterable
    to a line of the file as json."""
    with open(file_name, 'w+') as json_file:
        for item in dicts:
            json_file.write(json.dumps(item)+"\n")


def split_file_output_json(filename, dict_list, max_lines=1100):
    """
    Split an iterable of JSON serializable rows of data
     into groups and write each to a shard.
    """
    fill_object = object()
    groups = grouper(max_lines, dict_list, fillvalue=fill_object)
    dirname = os.path.dirname(filename)
    basename = os.path.basename(filename)
    for index, group in enumerate(groups):
        dict_group = takewhile(lambda x: x is not fill_object, group)
        dump_dicts_to_json_file(
            os.path.join(dirname, "{0}_{1}".format(index, basename)),<commit_msg>Add logical is not function<commit_after>from __future__ import print_function
import csv
from functools import partial
from itertools import takewhile
import json
import os
from itertools_recipes import grouper

LINE_BUFFER_SIZE = 5000


def ensure_dir(directory):
    """
    If directory doesn't exist, make it.

    :param directory: path to directory
    :type directory: `str`
    """
    if not os.path.exists(directory):
        os.makedirs(directory)


def is_not(a, b):
    """
    logical function to identify if a and b are not the same.
    :param a: any value
    :param b: any value
    """
    return a is not b


def i_get_csv_data(file_name, *args, **kwargs):
    """A generator for reading a csv file.
    """
    with open(file_name, 'r') as csv_file:
        reader = csv.reader(csv_file, *args, **kwargs)
        for row in reader:
            yield row


def write_as_csv(items, file_name, append=False, line_buffer_size=None):
    """
    Writes out items to a csv file in groups.

    :param items: An iterable collection of collections.
    :param file_name: path to the output file.
    :param append: whether to append or overwrite the file.
    :param line_buffer_size: number of lines to write at a time.
    """
    fill_object = object()
    is_not_fill = partial(is_not, fill_object)
    if line_buffer_size is None:
        line_buffer_size = LINE_BUFFER_SIZE
    if append:
        mode = 'a'
    else:
        mode = 'wt'
    with open(file_name, mode) as csv_file:
        writer = csv.writer(csv_file)
        line_groups = grouper(line_buffer_size,
                              items,
                              fillvalue=fill_object)

        for line_group in line_groups:
            writer.writerows(
                takewhile(is_not_fill, line_group))


def dump_dicts_to_json_file(file_name, dicts):
    """writes each dictionary in the dicts iterable
    to a line of the file as json."""
    with open(file_name, 'w+') as json_file:
        for item in dicts:
            json_file.write(json.dumps(item)+"\n")


def split_file_output_json(filename, dict_list, max_lines=1100):
    """
    Split an iterable of JSON serializable rows of data
     into groups and write each to a shard.
    """
    fill_object = object()
    groups = grouper(max_lines, dict_list, fillvalue=fill_object)
    dirname = os.path.dirname(filename)
    basename = os.path.basename(filename)
    for index, group in enumerate(groups):
        dict_group = takewhile(lambda x: x is not fill_object, group)
        dump_dicts_to_json_file(
            os.path.join(dirname, "{0}_{1}".format(index, basename)),
{'fraction_non_alphanumeric': 0.08891671884783969, 'fraction_numerical': 0.004174493842621582, 'mean_word_length': 2.720496894409938, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7611886', 'n_tokens_mistral': 1524, 'n_tokens_neox': 1472, 'n_words': 452}