<commit_before>from __future__ import absolute_import, division, print_function

import os
import shutil
from functools import partial
import numpy as np
import tables as tb
from blaze.utils import tmpfile

import datashape
from blaze.expr import (Selection, Head, Column, ColumnWise, Projection,
                        TableSymbol, Sort, Reduction, count)
from blaze.expr import eval_str
from blaze.compatibility import basestring, map
from datashape import Record
from ..dispatch import dispatch


@dispatch(tb.Table)
def discover(t):
    return t.shape[0] * Record([[col, t.coltypes[col]] for col in t.colnames])


@dispatch(tb.Table)
def drop(t):
    t.remove()


@dispatch(tb.Table, basestring)
def create_index(t, column, name=None, **kwargs):
    create_index(getattr(t.cols, column), **kwargs)


@dispatch(tb.Table, list)
def create_index(t, columns, name=None, **kwargs):
    if not all(map(partial(hasattr, t.cols), columns)):
        raise ValueError('table %s does not have all passed in columns %s' %
                         (t, columns))
    for column in columns:
        create_index(t, column, **kwargs)


@dispatch(tb.Column)
def create_index(c, optlevel=9, kind='full', name=None, **kwargs):
    c.create_index(optlevel=optlevel, kind=kind, **kwargs)


def to_dtype(dshape):
    dshape = datashape.dshape(dshape)
    dtype = datashape.to_numpy_dtype(dshape)
    fields = dict(dtype.fields.items())
    for k, (v, _) in fields.items():
        # pytables borks on unicode (even in Py3!) and object, probably should
        # raise here if either of those is the case
        if issubclass(v.type, basestring) and isinstance(dshape.subshape[k],
                                                         datashape.String):
            fields[k] = np.dtype('|S%d' % v.itemsize)
    return np.dtype(list(fields.items()))


def PyTables(path, datapath, dshape=None):
    def possibly_create_table(filename, dtype):
        f = tb.open_file(filename, mode='a')
        try:
            if datapath not in f:
                if dtype is None:
                    raise ValueError('dshape cannot be None and datapath not'
                                     ' in file')
                else:
                    f.create_table('/', datapath.lstrip('/'), description=dtype)
        finally:
            f.close()

    if dshape is not None:
        dtype = to_dtype(dshape)
    else:
        dtype = None

    if os.path.exists(path):
        possibly_create_table(path, dtype)
    else:
        with tmpfile('.h5') as filename:
            path = possibly_create_table(filename, dtype)
            shutil.copyfile(filename, path)
    return tb.open_file(path).get_node(datapath)


@dispatch(Selection, tb.Table)
def compute_up(sel, t, **kwargs):
    s = eval_str(sel.predicate.expr)
    return t.read_where(s)


@dispatch(TableSymbol, tb.Table)
def compute_up(ts, t, **kwargs):
    return t


@dispatch(Reduction, (tb.Column, tb.Table))
def compute_up(r, c, **kwargs):
    return compute_up(r, c[:])<commit_msg>Sort types function<commit_after>from __future__ import absolute_import, division, print_function

import os
import shutil
from functools import partial
import numpy as np
import tables as tb
from blaze.utils import tmpfile
from toolz import first, compose

import datashape
from blaze.expr import (Selection, Head, Column, ColumnWise, Projection,
                        TableSymbol, Sort, Reduction, count)
from blaze.expr import eval_str
from blaze.compatibility import basestring, map
from datashape import Record
from ..dispatch import dispatch


@dispatch(tb.Table)
def discover(t):
    return t.shape[0] * Record([[col, t.coltypes[col]] for col in t.colnames])


@dispatch(tb.Table)
def drop(t):
    t.remove()


@dispatch(tb.Table, basestring)
def create_index(t, column, name=None, **kwargs):
    create_index(getattr(t.cols, column), **kwargs)


@dispatch(tb.Table, list)
def create_index(t, columns, name=None, **kwargs):
    if not all(map(partial(hasattr, t.cols), columns)):
        raise ValueError('table %s does not have all passed in columns %s' %
                         (t, columns))
    for column in columns:
        create_index(t, column, **kwargs)


@dispatch(tb.Column)
def create_index(c, optlevel=9, kind='full', name=None, **kwargs):
    c.create_index(optlevel=optlevel, kind=kind, **kwargs)


def sort_dtype(items, names):
    return np.dtype(sorted(items, key=compose(names.index, first)))


def to_tables_descr(dtype):
    d = {}
    for (pos, name), (dtype, _) in zip(enumerate(dtype.names),
                                       map(dtype.fields.__getitem__,
                                           dtype.names)):
        if issubclass(dtype.type, np.datetime64):
            tdtype = tb.Description({name: tb.Time64Col(pos=pos)}),
        else:
            tdtype = tb.descr_from_dtype(np.dtype([(name, dtype)]))
        el = first(tdtype)
        getattr(el, name)._v_pos = pos
        d.update(el._v_colobjects)
    return d


def PyTables(path, datapath, dshape=None):
    def possibly_create_table(filename, dtype):
        f = tb.open_file(filename, mode='a')
        try:
            if datapath not in f:
                if dtype is None:
                    raise ValueError('dshape cannot be None and datapath not'
                                     ' in file')
                else:
                    f.create_table('/', datapath.lstrip('/'), description=dtype)
        finally:
            f.close()

    if dshape is not None:
        dtype = to_tables_descr(datashape.to_numpy_dtype(dshape))
    else:
        dtype = None

    if os.path.exists(path):
        possibly_create_table(path, dtype)
    else:
        with tmpfile('.h5') as filename:
            path = possibly_create_table(filename, dtype)
            shutil.copyfile(filename, path)
    return tb.open_file(path).get_node(datapath)


@dispatch(Selection, tb.Table)
def compute_up(sel, t, **kwargs):
    s = eval_str(sel.predicate.expr)
    return t.read_where(s)


@dispatch(TableSymbol, tb.Table)
def compute_up(ts, t, **kwargs):
    return t


@dispatch(Reduction, (tb.Column, tb.Table))
def compute_up(r, c, **kwargs):
    return compute_up(r, c[:])
{'fraction_non_alphanumeric': 0.1151798677206001, 'fraction_numerical': 0.0017744797547991611, 'mean_word_length': 2.743961352657005, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1041933', 'n_tokens_mistral': 1940, 'n_tokens_neox': 1922, 'n_words': 516}