<commit_before>#! /usr/bin/env python

""" Class used to handle multiset receiving and returning PL/Python procedures
"""

import re, skytools

from skytools import dbdict

__all__ = ['DBService', 'ServiceContext',
    'get_record', 'get_record_list',
    'make_record', 'make_record_array',
    #'log_result', 'transform_fields', 'TableAPI',
]

try:
    import plpy
except ImportError:
    pass

def transform_fields(rows, key_fields, name_field, data_field):
    """Convert multiple-rows per key input array
    to one-row, multiple-column output array.  The input arrays
    must be sorted by the key fields.
    
    >>> rows = []
    >>> rows.append({'time': '22:00', 'metric': 'count', 'value': 100})
    >>> rows.append({'time': '22:00', 'metric': 'dur', 'value': 7})
    >>> rows.append({'time': '23:00', 'metric': 'count', 'value': 200})
    >>> rows.append({'time': '23:00', 'metric': 'dur', 'value': 5})
    >>> transform_fields(rows, ['time'], 'metric', 'value')
    [{'count': 100, 'dur': 7, 'time': '22:00'}, {'count': 200, 'dur': 5, 'time': '23:00'}]
    """
    cur_key = None
    cur_row = None
    res = []
    for r in rows:
        k = [r[f] for f in key_fields]
        if k != cur_key:
            cur_key = k
            cur_row = {}
            for f in key_fields:
                cur_row[f] = r[f]
            res.append(cur_row)
        cur_row[r[name_field]] = r[data_field]
    return res

# render_table
def render_table(rows, fields):
    """ Render result rows as a table.
        Returns array of lines.
    """
    widths = [15] * len(fields)
    for row in rows:
        for i, k in enumerate(fields):
            rlen = len(str(row.get(k)))
            widths[i] = widths[i] > rlen and widths[i] or rlen
    widths = [w + 2 for w in widths]

    fmt = '%%-%ds' * (len(widths) - 1) + '%%s'
    fmt = fmt % tuple(widths[:-1])

    lines = []
    lines.append(fmt % tuple(fields))
    lines.append(fmt % tuple(['-'*15] * len(fields)))
    for row in rows:
        lines.append(fmt % tuple([str(row.get(k)) for k in fields]))
    return lines

# data conversion to and from url

def get_record(arg):
    """ Parse data for one urlencoded record.
        Useful for turning incoming serialized data into structure usable for manipulation.
    """
    if not arg:
        return dbdict()

    # allow array of single record
    if arg[0] in ('{', '['):
        lst = skytools.parse_pgarray(arg)
        if len(lst) != 1:
            raise ValueError('get_record() expects exactly 1 row, got %d' % len(lst))
        arg = lst[0]

    # parse record
    return dbdict(skytools.db_urldecode(arg))

def get_record_list(array):
    """ Parse array of urlencoded records.
        Useful for turning incoming serialized data into structure usable for manipulation.
    """
    if array is None:
        return []
    return map(get_record, skytools.parse_pgarray(array))

def get_record_lists(tbl, field):
    """ Create dictionary of lists from given list using field as grouping criteria
        Used for master detail operatons to group detail records according to master id
    """
    dict = dbdict()
    for rec in tbl:
        id = str( rec[field] )
        dict.setdefault( id, [] ).append(rec)
    return dict

def _make_record_convert(row):
    """Converts complex values."""
    d = row.copy()
    for k, v in d.items():<commit_msg>dbservice.get_record_list: do not use parse_pgarray on python lists<commit_after>#! /usr/bin/env python

""" Class used to handle multiset receiving and returning PL/Python procedures
"""

import re, skytools

from skytools import dbdict

__all__ = ['DBService', 'ServiceContext',
    'get_record', 'get_record_list',
    'make_record', 'make_record_array',
    #'log_result', 'transform_fields', 'TableAPI',
]

try:
    import plpy
except ImportError:
    pass

def transform_fields(rows, key_fields, name_field, data_field):
    """Convert multiple-rows per key input array
    to one-row, multiple-column output array.  The input arrays
    must be sorted by the key fields.
    
    >>> rows = []
    >>> rows.append({'time': '22:00', 'metric': 'count', 'value': 100})
    >>> rows.append({'time': '22:00', 'metric': 'dur', 'value': 7})
    >>> rows.append({'time': '23:00', 'metric': 'count', 'value': 200})
    >>> rows.append({'time': '23:00', 'metric': 'dur', 'value': 5})
    >>> transform_fields(rows, ['time'], 'metric', 'value')
    [{'count': 100, 'dur': 7, 'time': '22:00'}, {'count': 200, 'dur': 5, 'time': '23:00'}]
    """
    cur_key = None
    cur_row = None
    res = []
    for r in rows:
        k = [r[f] for f in key_fields]
        if k != cur_key:
            cur_key = k
            cur_row = {}
            for f in key_fields:
                cur_row[f] = r[f]
            res.append(cur_row)
        cur_row[r[name_field]] = r[data_field]
    return res

# render_table
def render_table(rows, fields):
    """ Render result rows as a table.
        Returns array of lines.
    """
    widths = [15] * len(fields)
    for row in rows:
        for i, k in enumerate(fields):
            rlen = len(str(row.get(k)))
            widths[i] = widths[i] > rlen and widths[i] or rlen
    widths = [w + 2 for w in widths]

    fmt = '%%-%ds' * (len(widths) - 1) + '%%s'
    fmt = fmt % tuple(widths[:-1])

    lines = []
    lines.append(fmt % tuple(fields))
    lines.append(fmt % tuple(['-'*15] * len(fields)))
    for row in rows:
        lines.append(fmt % tuple([str(row.get(k)) for k in fields]))
    return lines

# data conversion to and from url

def get_record(arg):
    """ Parse data for one urlencoded record.
        Useful for turning incoming serialized data into structure usable for manipulation.
    """
    if not arg:
        return dbdict()

    # allow array of single record
    if arg[0] in ('{', '['):
        lst = skytools.parse_pgarray(arg)
        if len(lst) != 1:
            raise ValueError('get_record() expects exactly 1 row, got %d' % len(lst))
        arg = lst[0]

    # parse record
    return dbdict(skytools.db_urldecode(arg))

def get_record_list(array):
    """ Parse array of urlencoded records.
        Useful for turning incoming serialized data into structure usable for manipulation.
    """
    if array is None:
        return []

    if isinstance(array, list):
        return map(get_record, array)
    else:
        return map(get_record, skytools.parse_pgarray(array))

def get_record_lists(tbl, field):
    """ Create dictionary of lists from given list using field as grouping criteria
        Used for master detail operatons to group detail records according to master id
    """
    dict = dbdict()
    for rec in tbl:
        id = str( rec[field] )
        dict.setdefault( id, [] ).append(rec)
    return dict

def _make_record_convert(row):
    """Converts complex values."""
    d = row.copy()
    for k, v in d.items():
{'fraction_non_alphanumeric': 0.14291951680978024, 'fraction_numerical': 0.014845000727696114, 'mean_word_length': 2.732753938077132, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5872993', 'n_tokens_mistral': 2290, 'n_tokens_neox': 2188, 'n_words': 695}