<commit_before># encoding: utf-8
#----------------------------------------------------------------------------
#  Copyright (C) 2008-2014, IPython Development Team and Enthought, Inc.
#  Distributed under the terms of the BSD License.  See COPYING.rst.
#----------------------------------------------------------------------------

"""
Plot distributions for some distarrays for the protocol documentation.

The output from this example program should be directly usable in the
distributed array protocol documentation, if saved as examples.rst.
The output .png files should be copied to the images folder as well.
"""

from __future__ import print_function

import os.path
from pprint import pformat
from numpy.random import permutation, seed

import distarray
from distarray import plotting


def print_array_documentation(context,
                              array,
                              title,
                              text,
                              global_plot_filename,
                              local_plot_filename,
                              verbose=False):
    """ Print some properties of the array.

    The output is rst formatted, so that it can be directly
    used for documentation. It includes a plot of the array distribution,
    some properties that are same for each process, and properties that
    vary per process.

    Parameters
    ----------
    context : distarray.Context
        The context that will be used to query the array properties.
    array : DistArray
        The array to describe.
    title : string
        The document section title.
    text : string
        A description of the array layout to add to the document.
    global_plot_filename : string
        The filename of the global array figure to add to the document.
    local_plot_filename : string
        The filename of the local array figure to add to the document.
    verbose : bool
        If True, some extra information is printed, that would
        be overly verbose for the real protocol documentation.
    """

    def rst_print(obj):
        """ Return text that formats obj nicely for an .rst document. """
        text = pformat(obj)
        # pformat() gives blank lines for 3d arrays, which confuse Sphinx.
        lines = text.split('\n')
        trims = [line for line in lines if len(line) > 0]
        trimmed = '\n'.join(trims)
        return trimmed

    def rst_plot(filename):
        """ Include a plot in the .rst document. """
        print(".. image:: ../images/%s" % (filename))
        # align right does not work as I want.
        #print("   :align: right")
        print()

    # Examine the array on all the engines.
    cmd = 'distbuffer = %s.__distarray__()' % (array.key)
    context._execute(cmd)
    cmd = 'db_keys = distbuffer.keys()'
    context._execute(cmd)
    cmd = 'db_version = distbuffer["__version__"]'
    context._execute(cmd)
    cmd = 'db_buffer = distbuffer["buffer"]'
    context._execute(cmd)
    cmd = 'db_dim_data = distbuffer["dim_data"]'
    context._execute(cmd)
    # Get data from each engine.
    db_keys = context.view['db_keys']
    db_version = context.view['db_version']
    db_buffer = context.view['db_buffer']
    db_dim_data = context.view['db_dim_data']

    print("%s" % (title))
    print("%s" % ('`' * len(title)))
    print()
    print(text)
    print()

    # Global array plot.
    if global_plot_filename is not None:
        rst_plot(global_plot_filename)

    # Full (undistributed) array:
    full_array = array.toarray()
    print("The full (undistributed) array:")
    print()
    print(">>> full_array")
    print(rst_print(full_array))
    print()

    # Result of get_localarrays() and get_ndarrays().
    # This is mainly for debugging and will eventually not be needed.
    if verbose:
        local_arrays = array.get_localarrays()
        nd_arrays = array.get_ndarrays()
        print("Result of get_localarrays() and get_ndarrays():")
        print()
        print(">>> get_localarrays()")
        print(rst_print(local_arrays))
        print(">>> get_ndarrays()")
        print(rst_print(nd_arrays))
        print()

    # Properties that are the same on all processes:
    print("In all processes:")
    print()
    print(">>> distbuffer = local_array.__distarray__()")
    print(">>> distbuffer.keys()")
    print(rst_print(db_keys[0]))
    print(">>> distbuffer['__version__']")
    print(rst_print(db_version[0]))
    print()

    # Local array plot.
    if local_plot_filename is not None:
        rst_plot(local_plot_filename)

    # Properties that change per-process:
    for p, (keys, version, buffer, dim_data) in enumerate(
            zip(db_keys, db_version, db_buffer, db_dim_data)):
        print("In process %d:" % (p))
        print()
        print(">>> distbuffer['buffer']")
        print(rst_print(buffer))
        print(">>> distbuffer['dim_data']")
        print(rst_print(dim_data))
        print()


def create_distribution_plot_and_documentation(context, params):
    """ Create an array distribution plot,
    and the related .rst documentation.
    """

    def shape_text(shape):
        """ Get a text string describing the array shape. """
        # Always want to display at least N X M.
        if len(shape) == 1:
            shape = (1, shape[0])
        shape_labels = ['%d' % (s) for s in shape]<commit_msg>Writing per-engine properties into rst grid table, and skipping those with empty local arrays.<commit_after># encoding: utf-8
#----------------------------------------------------------------------------
#  Copyright (C) 2008-2014, IPython Development Team and Enthought, Inc.
#  Distributed under the terms of the BSD License.  See COPYING.rst.
#----------------------------------------------------------------------------

"""
Plot distributions for some distarrays for the protocol documentation.

The output from this example program should be directly usable in the
distributed array protocol documentation, if saved as examples.rst.
The output .png files should be copied to the images folder as well.
"""

from __future__ import print_function

import os.path
from pprint import pformat
from numpy import empty
from numpy.random import permutation, seed

import distarray
from distarray import plotting


def print_array_documentation(context,
                              array,
                              title,
                              text,
                              global_plot_filename,
                              local_plot_filename,
                              verbose=False):
    """ Print some properties of the array.

    The output is rst formatted, so that it can be directly
    used for documentation. It includes a plot of the array distribution,
    some properties that are same for each process, and properties that
    vary per process.

    Parameters
    ----------
    context : distarray.Context
        The context that will be used to query the array properties.
    array : DistArray
        The array to describe.
    title : string
        The document section title.
    text : string
        A description of the array layout to add to the document.
    global_plot_filename : string
        The filename of the global array figure to add to the document.
    local_plot_filename : string
        The filename of the local array figure to add to the document.
    verbose : bool
        If True, some extra information is printed, that would
        be overly verbose for the real protocol documentation.
    """

    def rst_lines(obj):
        """ Return lines of text that format obj for an .rst document. """
        text = pformat(obj)
        # pformat() gives blank lines for 3d arrays, which confuse Sphinx.
        lines = text.split('\n')
        trims = [line for line in lines if len(line) > 0]
        return trims

    def rst_print(obj):
        """ Return text that formats obj nicely for an .rst document. """
        lines = rst_lines(obj)
        text = '\n'.join(lines)
        return text

    def rst_plot(filename):
        """ Include a plot in the .rst document. """
        print(".. image:: ../images/%s" % (filename))
        # align right does not work as I want.
        #print("   :align: right")
        print()

    def text_block_size(lines):
        """ Determine the number of rows and columns to print lines. """
        line_count = len(lines)
        line_width = max([len(line) for line in lines])
        return line_count, line_width

    def text_block_max_size(lines_list):
        """ Determine  number of rows/columns for the largest line list. """
        block_size = empty((len(lines_list), 2), dtype=int)
        for itext, lines in enumerate(lines_list):
            line_count, line_width = text_block_size(lines)
            block_size[itext, 0] = line_count
            block_size[itext, 1] = line_width
        #print('Table sizes:')
        #print(block_size)
        max_size = block_size.max(axis=0)
        #print('Max size:')
        #print(max_size)
        max_rows, max_cols = max_size[0], max_size[1]
        return max_rows, max_cols

    def rst_table(rows, cols, lines_list):
        """ Print the list of lines as a .rst table. """
        num_cells = rows * cols
        num_texts = len(lines_list)
        if num_cells != num_texts:
            raise ValueError('Invalid table size %d x %d for %d entries.' % (
                rows, cols, num_texts))
        # Determine table size needed for biggest text blocks.
        max_sizes = text_block_max_size(lines_list)
        #print('max_sizes:')
        #print(max_sizes)
        max_lines, max_cols = max_sizes
        # Table row separator.
        sep = '-' * max_cols
        seps = [sep for i in range(cols)]
        header = '+' + '+'.join(seps) + '+'
        # Group text blocks into array pattern.
        text_blocks = []
        print(header)
        for row in range(rows):
            for line in range(max_lines):
                col_lines = []
                for col in range(cols):
                    iblock = row * cols + col
                    lines = lines_list[iblock]
                    if line < len(lines):
                        text = lines[line]
                    else:
                        text = ''
                    text = text.ljust(max_cols)
                    col_lines.append(text)
                col_line = '|' + '|'.join(col_lines) + '|'
                print(col_line)
            print(header)
        #print('text_blocks:')
        #print(text_blocks)
        print()
        # Non-table layout.
        for lines in lines_list:
            for line in lines:
                print(line)
            print()

    # Examine the array on all the engines.
    cmd = 'distbuffer = %s.__distarray__()' % (array.key)
    context._execute(cmd)
    cmd = 'db_keys = distbuffer.keys()'
    context._execute(cmd)
    cmd = 'db_version = distbuffer["__version__"]'
    context._execute(cmd)
    cmd = 'db_buffer = distbuffer["buffer"]'
    context._execute(cmd)
    cmd = 'db_dim_data = distbuffer["dim_data"]'
    context._execute(cmd)
    # Get data from each engine.
    db_keys = context.view['db_keys']
    db_version = context.view['db_version']
    db_buffer = context.view['db_buffer']
    db_dim_data = context.view['db_dim_data']
    # Get local ndarrays.
    db_ndarrays = array.get_ndarrays()

    print("%s" % (title))
    print("%s" % ('`' * len(title)))
    print()
    print(text)
    print()

    # Global array plot.
    if global_plot_filename is not None:
        rst_plot(global_plot_filename)

    # Full (undistributed) array:
    full_array = array.toarray()
    print("The full (undistributed) array:")
    print()
    print(">>> full_array")
    print(rst_print(full_array))
    print()

    # Result of get_localarrays() and get_ndarrays().
    # This is mainly for debugging and will eventually not be needed.
    if verbose:
        local_arrays = array.get_localarrays()
        nd_arrays = array.get_ndarrays()
        print("Result of get_localarrays() and get_ndarrays():")
        print()
        print(">>> get_localarrays()")
        print(rst_print(local_arrays))
        print(">>> get_ndarrays()")
        print(rst_print(nd_arrays))
        print()

    # Properties that are the same on all processes:
    print("In all processes:")
    print()
    print(">>> distbuffer = local_array.__distarray__()")
    print(">>> distbuffer.keys()")
    print(rst_print(db_keys[0]))
    print(">>> distbuffer['__version__']")
    print(rst_print(db_version[0]))
    print()

    # Local array plot.
    if local_plot_filename is not None:
        rst_plot(local_plot_filename)

    # Properties that change per-process:
    lines_list = []
    for p, (keys, version, buffer, dim_data, ndarray) in enumerate(
            zip(db_keys, db_version, db_buffer, db_dim_data, db_ndarrays)):
        # Skip if local ndarray is empty, as there is no local plot.
        if ndarray.size == 0:
            continue
        lines = []
        lines += ["In process %d:" % (p), ""]
        lines += [">>> distbuffer['buffer']"] + rst_lines(buffer)
        lines += [">>> distbuffer['dim_data']"] + rst_lines(dim_data)
        #lines += [""]
        lines_list.append(lines)
    # Print as table with nice layout.
    num_local_properties = len(lines_list)
    if (num_local_properties % 2) == 0:
        # 2 X N grid
        rows, cols = (2, num_local_properties // 2)
    else:
        # N x 1 grid
        rows, cols = (num_local_properties, 1)
    rst_table(rows, cols, lines_list)


def create_distribution_plot_and_documentation(context, params):
    """ Create an array distribution plot,
    and the related .rst documentation.
    """

    def shape_text(shape):
        """ Get a text string describing the array shape. """
        # Always want to display at least N X M.
        if len(shape) == 1:
            shape = (1, shape[0])
        shape_labels = ['%d' % (s) for s in shape]
{'fraction_non_alphanumeric': 0.120963132323521, 'fraction_numerical': 0.003286653329522721, 'mean_word_length': 2.1624491640307277, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4022677', 'n_tokens_mistral': 4004, 'n_tokens_neox': 3830, 'n_words': 1267}