<commit_before># coding: utf-8

"""
PyArrow related utilities.
"""

__all__ = ["merge_parquet_files", "merge_parquet_task"]


import os
import shutil

import six

from law.target.local import LocalFileTarget, LocalDirectoryTarget
from law.util import map_verbose, human_bytes


def merge_parquet_files(src_paths, dst_path, force=True, callback=None, writer_opts=None):
    """
    Merges parquet files in *src_paths* into a new file at *dst_path*. Intermediate directories are
    created automatically. When *dst_path* exists and *force* is *True*, the file is removed first.
    Otherwise, an exception is thrown. *callback* can refer to a callable accepting a single integer
    argument representing the index of the file after it was merged. *writer_opts* can be a
    dictionary of keyword arguments that are passed to the *ParquetWriter* instantiation. Its
    defaults are:

        - "version": "2.0"
        - "compression": "gzip"
        - "use_dictionary": True
        - "data_page_size": 2097152
        - "write_statistics": True

    The full, expanded *dst_path* is returned.
    """
    import pyarrow.parquet as pq

    if not src_paths:
        raise Exception("cannot merge empty list of parquet files")

    # default writer options
    _writer_opts = dict(
        version="2.0",
        compression="gzip",
        use_dictionary=True,
        data_page_size=2097152,
        write_statistics=True,
    )
    _writer_opts.update(writer_opts or {})

    # default callable
    if not callable(callback):
        callback = lambda i: None

    # prepare the dst directory
    dst_path = os.path.expandvars(os.path.expanduser(dst_path))
    dir_name = os.path.dirname(dst_path)
    if not os.path.exists(dir_name):
        os.makedirs(dir_name)

    # remove the file first when existing<commit_msg>Fix parquet writer version in pyarrow contrib.<commit_after># coding: utf-8

"""
PyArrow related utilities.
"""

__all__ = ["merge_parquet_files", "merge_parquet_task"]


import os
import shutil

import six

from law.target.local import LocalFileTarget, LocalDirectoryTarget
from law.util import map_verbose, human_bytes


def merge_parquet_files(src_paths, dst_path, force=True, callback=None, writer_opts=None):
    """
    Merges parquet files in *src_paths* into a new file at *dst_path*. Intermediate directories are
    created automatically. When *dst_path* exists and *force* is *True*, the file is removed first.
    Otherwise, an exception is thrown. *callback* can refer to a callable accepting a single integer
    argument representing the index of the file after it was merged. *writer_opts* can be a
    dictionary of keyword arguments that are passed to the *ParquetWriter* instantiation. Its
    defaults are:

        - "version": "2.0"
        - "compression": "gzip"
        - "use_dictionary": True
        - "data_page_size": 2097152
        - "write_statistics": True

    The full, expanded *dst_path* is returned.
    """
    import pyarrow.parquet as pq

    if not src_paths:
        raise Exception("cannot merge empty list of parquet files")

    # default writer options
    _writer_opts = dict(
        version="2.4",
        compression="gzip",
        use_dictionary=True,
        data_page_size=2097152,
        write_statistics=True,
    )
    _writer_opts.update(writer_opts or {})

    # default callable
    if not callable(callback):
        callback = lambda i: None

    # prepare the dst directory
    dst_path = os.path.expandvars(os.path.expanduser(dst_path))
    dir_name = os.path.dirname(dst_path)
    if not os.path.exists(dir_name):
        os.makedirs(dir_name)

    # remove the file first when existing
{'fraction_non_alphanumeric': 0.08517006802721089, 'fraction_numerical': 0.010340136054421769, 'mean_word_length': 3.2448036951501154, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6858229', 'n_tokens_mistral': 1116, 'n_tokens_neox': 1076, 'n_words': 351}