<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
""" An extensible ASCII table reader and writer.

core.py:
  Core base classes and functions for reading and writing tables.

:Copyright: Smithsonian Astrophysical Observatory (2010)
:Author: <NAME> (<EMAIL>)
"""

from __future__ import absolute_import, division, print_function

import os
import re
import csv
import itertools
import functools
import numpy
import warnings
import copy

from ...extern import six
from ...extern.six.moves import zip
from ...extern.six.moves import cStringIO as StringIO
from ...utils.exceptions import AstropyWarning

from ...table import Table
from ...table.column import col_getattr, col_setattr, col_iter_str_vals
from ...utils.compat import ignored
from ...utils.data import get_readable_fileobj
from ...utils import OrderedDict
from . import connect

# Global dictionary mapping format arg to the corresponding Reader class
FORMAT_CLASSES = {}

# Similar dictionary for fast readers
FAST_CLASSES = {}

class MaskedConstant(numpy.ma.core.MaskedConstant):
    """A trivial extension of numpy.ma.masked

    We want to be able to put the generic term ``masked`` into a dictionary.
    In python 2.7 we can just use ``numpy.ma.masked``, but in python 3.1 and 3.2 that
    is not hashable, see https://github.com/numpy/numpy/issues/4660
    So, we need to extend it here with a hash value.
    """
    def __hash__(self):
        '''All instances of this class shall have the same hash.'''
        # Any large number will do.
        return 1234567890

masked = MaskedConstant()

class InconsistentTableError(ValueError):
    """
    Indicates that an input table is inconsistent in some way.

    The default behavior of ``BaseReader`` is to throw an instance of
    this class if a data row doesn't match the header.
    """

class OptionalTableImportError(ImportError):
    """
    Indicates that a dependency for table reading is not present.

    An instance of this class is raised whenever an optional reader
    with certain required dependencies cannot operate because of
    an ImportError.
    """

class ParameterError(NotImplementedError):
    """
    Indicates that a reader cannot handle a passed parameter.

    The C-based fast readers in ``io.ascii`` raise an instance of
    this error class upon encountering a parameter that the
    C engine cannot handle.
    """

class FastOptionsError(NotImplementedError):
    """
    Indicates that one of the specified options for fast
    reading is invalid.
    """

class NoType(object):
    """
    Superclass for ``StrType`` and ``NumType`` classes.

    This class is the default type of ``Column`` and provides a base
    class for other data types.
    """


class StrType(NoType):
    """
    Indicates that a column consists of text data.
    """


class NumType(NoType):
    """
    Indicates that a column consists of numerical data.
    """


class FloatType(NumType):
    """
    Describes floating-point data.
    """


class BoolType(NoType):
    """
    Describes boolean data.
    """


class IntType(NumType):
    """
    Describes integer data.
    """


class AllType(StrType, FloatType, IntType):
    """
    Subclass of all other data types.

    This type is returned by ``convert_numpy`` if the given numpy
    type does not match ``StrType``, ``FloatType``, or ``IntType``.
    """


class Column(object):
    """Table column.

    The key attributes of a Column object are:

    * **name** : column name
    * **type** : column type (NoType, StrType, NumType, FloatType, IntType)
    * **dtype** : numpy dtype (optional, overrides **type** if set)
    * **str_vals** : list of column values as strings
    * **data** : list of converted column values
    """
    def __init__(self, name):
        self.name = name
        self.type = NoType  # Generic type (Int, Float, Str etc)
        self.dtype = None  # Numpy dtype if available
        self.str_vals = []
        self.fill_values = {}


class BaseInputter(object):
    """
    Get the lines from the table input and return a list of lines.

    """
    def get_lines(self, table):
        """
        Get the lines from the ``table`` input. The input table can be one of:

        * File name
        * String (newline separated) with all header and data lines (must have at least 2 lines)
        * File-like object with read() method
        * List of strings

        Parameters
        ----------
        table : str, file_like, list
            Can be either a file name, string (newline separated) with all header and data
            lines (must have at least 2 lines), a file-like object with a ``read()`` method,
            or a list of strings.

        Returns
        -------
        lines : list
            List of lines
        """
        try:
            if (hasattr(table, 'read') or
                    ('\n' not in table + '' and '\r' not in table + '')):
                with get_readable_fileobj(table) as file_obj:
                    table = file_obj.read()
            lines = table.splitlines()
        except TypeError:
            try:
                # See if table supports indexing, slicing, and iteration
                table[0]
                table[0:1]
                iter(table)
                lines = table
            except TypeError:
                raise TypeError(
                    'Input "table" must be a string (filename or data) or an iterable')

        return self.process_lines(lines)

    def process_lines(self, lines):
        """Process lines for subsequent use.  In the default case do nothing.
        This routine is not generally intended for removing comment lines or
        stripping whitespace.  These are done (if needed) in the header and
        data line processing.

        Override this method if something more has to be done to convert raw
        input lines to the table rows.  For example the
        ContinuationLinesInputter derived class accounts for continuation
        characters if a row is split into lines."""
        return lines


class BaseSplitter(object):
    """
    Base splitter that uses python's split method to do the work.

    This does not handle quoted values.  A key feature is the formulation of
    __call__ as a generator that returns a list of the split line values at
    each iteration.

    There are two methods that are intended to be overridden, first
    ``process_line()`` to do pre-processing on each input line before splitting
    and ``process_val()`` to do post-processing on each split string value.  By
    default these apply the string ``strip()`` function.  These can be set to
    another function via the instance attribute or be disabled entirely, for
    example::

      reader.header.splitter.process_val = lambda x: x.lstrip()
      reader.data.splitter.process_val = None

    """

    #: one-character string used to separate fields
    delimiter = None

    def process_line(self, line):
        """Remove whitespace at the beginning or end of line.  This is especially useful for
        whitespace-delimited files to prevent spurious columns at the beginning or end."""
        return line.strip()

    def process_val(self, val):
        """Remove whitespace at the beginning or end of value."""
        return val.strip()

    def __call__(self, lines):
        if self.process_line:
            lines = (self.process_line(x) for x in lines)
        for line in lines:
            vals = line.split(self.delimiter)
            if self.process_val:
                yield [self.process_val(x) for x in vals]
            else:
                yield vals

    def join(self, vals):
        if self.delimiter is None:
            delimiter = ' '
        else:
            delimiter = self.delimiter
        return delimiter.join(str(x) for x in vals)


class DefaultSplitter(BaseSplitter):
    """Default class to split strings into columns using python csv.  The class
    attributes are taken from the csv Dialect class.

    Typical usage::

      # lines = ..
      splitter = ascii.DefaultSplitter()
      for col_vals in splitter(lines):
          for col_val in col_vals:
               ...

    """
    delimiter = ' '  #: one-character string used to separate fields.
    quotechar = '"'  #: control how instances of *quotechar* in a field are quoted
    doublequote = True  #: character to remove special meaning from following character
    escapechar = None  #: one-character stringto quote fields containing special characters
    quoting = csv.QUOTE_MINIMAL  #: control when quotes are recognised by the reader
    skipinitialspace = True  #: ignore whitespace immediately following the delimiter
    csv_writer = None
    csv_writer_out = StringIO()


    def process_line(self, line):
        """Remove whitespace at the beginning or end of line.  This is especially useful for
        whitespace-delimited files to prevent spurious columns at the beginning or end.
        If splitting on whitespace then replace unquoted tabs with space first"""
        if self.delimiter == '\s':
            line = _replace_tab_with_space(line, self.escapechar, self.quotechar)
        return line.strip()


    def __call__(self, lines):
        """Return an iterator over the table ``lines``, where each iterator output
        is a list of the split line values.

        Parameters
        ----------
        lines : list
            List of table lines

        Returns
        -------
        lines : iterator

        """
        if self.process_line:
            lines = [self.process_line(x) for x in lines]

        # In Python 2.x the inputs to csv cannot be unicode.  In Python 3 these
        # lines do nothing.
        escapechar = None if self.escapechar is None else str(self.escapechar)
        quotechar = None if self.quotechar is None else str(self.quotechar)
        delimiter = None if self.delimiter is None else str(self.delimiter)

        if delimiter == '\s':
            delimiter = ' '

        csv_reader = csv.reader(lines,
                                delimiter=delimiter,
                                doublequote=self.doublequote,
                                escapechar=escapechar,
                                quotechar=quotechar,
                                quoting=self.quoting,
                                skipinitialspace=self.skipinitialspace
                                )
        for vals in csv_reader:
            if self.process_val:
                yield [self.process_val(x) for x in vals]
            else:
                yield vals

    def join(self, vals):

        # In Python 2.x the inputs to csv cannot be unicode
        escapechar = None if self.escapechar is None else str(self.escapechar)
        quotechar = None if self.quotechar is None else str(self.quotechar)
        delimiter = ' ' if self.delimiter is None else str(self.delimiter)

        if self.csv_writer is None:
            self.csv_writer = csv.writer(self.csv_writer_out,
                                         delimiter=delimiter,
                                         doublequote=self.doublequote,
                                         escapechar=escapechar,
                                         quotechar=quotechar,
                                         quoting=self.quoting,
                                         lineterminator='',
                                         )
        self.csv_writer_out.seek(0)
        self.csv_writer_out.truncate()
        if self.process_val:
            vals = [self.process_val(x) for x in vals]
        self.csv_writer.writerow(vals)

        return self.csv_writer_out.getvalue()


def _replace_tab_with_space(line, escapechar, quotechar):
    """Replace tab with space within ``line`` while respecting quoted substrings"""
    newline = []
    in_quote = False
    lastchar = 'NONE'
    for char in line:
        if char == quotechar and lastchar != escapechar:
            in_quote = not in_quote
        if char == '\t' and not in_quote:
            char = ' '
        lastchar = char
        newline.append(char)
    return ''.join(newline)


def _get_line_index(line_or_func, lines):
    """Return the appropriate line index, depending on ``line_or_func`` which
    can be either a function, a positive or negative int, or None.
    """

    if hasattr(line_or_func, '__call__'):
        return line_or_func(lines)
    elif line_or_func:
        if line_or_func >= 0:
            return line_or_func
        else:
            n_lines = sum(1 for line in lines)
            return n_lines + line_or_func
    else:
        return line_or_func


class BaseHeader(object):
    """
    Base table header reader
    """
    auto_format = 'col%d'  #: format string for auto-generating column names
    start_line = None  #: None, int, or a function of ``lines`` that returns None or int
    comment = None  #: regular expression for comment lines
    splitter_class = DefaultSplitter  #: Splitter class for splitting data lines into columns
    names = None  #: list of names corresponding to each data column
    write_comment = False
    write_spacer_lines = ['ASCII_TABLE_WRITE_SPACER_LINE']

    def __init__(self):
        self.splitter = self.splitter_class()

    def _set_cols_from_names(self):
        self.cols = [Column(name=x) for x in self.names]

    def update_meta(self, lines, meta):
        """
        Extract any table-level metadata, e.g. keywords, comments, column metadata, from
        the table ``lines`` and update the OrderedDict ``meta`` in place.  This base
        method extracts comment lines and stores them in ``meta`` for output.
        """
        if self.comment:
            re_comment = re.compile(self.comment)
            comment_lines = [x for x in lines if re_comment.match(x)]
        else:
            comment_lines = []
        comment_lines = [re.sub('^' + self.comment, '', x).strip()
                         for x in comment_lines]
        if comment_lines:
            meta.setdefault('table', {})['comments'] = comment_lines

    def get_cols(self, lines):
        """Initialize the header Column objects from the table ``lines``.

        Based on the previously set Header attributes find or create the column names.
        Sets ``self.cols`` with the list of Columns.

        Parameters
        ----------
        lines : list
            List of table lines

        """

        start_line = _get_line_index(self.start_line, self.process_lines(lines))
        if start_line is None:
            # No header line so auto-generate names from n_data_cols
            # Get the data values from the first line of table data to determine n_data_cols
            try:
                first_data_vals = next(self.data.get_str_vals())
            except StopIteration:
                raise InconsistentTableError('No data lines found so cannot autogenerate '
                                             'column names')
            n_data_cols = len(first_data_vals)
            self.names = [self.auto_format % i for i in range(1, n_data_cols + 1)]

        else:
            for i, line in enumerate(self.process_lines(lines)):
                if i == start_line:
                    break
            else:  # No header line matching
                raise ValueError('No header line found in table')

            self.names = next(self.splitter([line]))

        self._set_cols_from_names()

    def process_lines(self, lines):
        """Generator to yield non-comment lines"""
        if self.comment:
            re_comment = re.compile(self.comment)
        # Yield non-comment lines
        for line in lines:
            if line and (not self.comment or not re_comment.match(line)):
                yield line

    def write_comments(self, lines, meta):
        if self.write_comment is not False:
            for comment in meta.get('comments', []):
                lines.append(self.write_comment + comment)

    def write(self, lines):
        if self.start_line is not None:
            for i, spacer_line in zip(range(self.start_line),
                                      itertools.cycle(self.write_spacer_lines)):
                lines.append(spacer_line)
            lines.append(self.splitter.join([col_getattr(x, 'name') for x in self.cols]))

    @property
    def colnames(self):
        """Return the column names of the table"""
        return tuple(col.name if isinstance(col, Column) else col_getattr(col, 'name')
                     for col in self.cols)

    def get_type_map_key(self, col):
        return col.raw_type

    def get_col_type(self, col):
        try:
            type_map_key = self.get_type_map_key(col)
            return self.col_type_map[type_map_key.lower()]
        except KeyError:
            raise ValueError('Unknown data type ""%s"" for column "%s"' % (
                col.raw_type, col.name))

    def check_column_names(self, names, strict_names, guessing):
        """
        Check column names.

        This must be done before applying the names transformation
        so that guessing will fail appropriately if ``names`` is supplied.
        For instance if the basic reader is given a table with no column header
        row.

        Parameters
        ----------
        names : list
            User-supplied list of column names
        strict_names : bool
            Whether to impose extra requirements on names
        guessing : bool
            TODO: ?
        """
        if strict_names:
            # Impose strict requirements on column names (normally used in guessing)
            bads = [" ", ",", "|", "\t", "'", '"']
            for name in self.colnames:
                if (_is_number(name) or
                    len(name) == 0 or
                    name[0] in bads or
                    name[-1] in bads):
                    raise ValueError('Column name {0!r} does not meet strict name requirements'
                                     .format(name))
        # When guessing require at least two columns
        if guessing and len(self.colnames) <= 1:
            raise ValueError

        if names is not None and len(names) != len(self.colnames):
            raise ValueError('Length of names argument ({0}) does not match number'
                             ' of table columns ({1})'.format(len(names), len(self.colnames)))


class BaseData(object):
    """
    Base table data reader.
    """
    start_line = None  #: None, int, or a function of ``lines`` that returns None or int
    end_line = None  #: None, int, or a function of ``lines`` that returns None or int
    comment = None  #: Regular expression for comment lines
    splitter_class = DefaultSplitter  #: Splitter class for splitting data lines into columns
    write_spacer_lines = ['ASCII_TABLE_WRITE_SPACER_LINE']
    fill_include_names = None
    fill_exclude_names = None
    # Currently, the default matches the numpy default for masked values.
    fill_values = [(masked, '--')]
    formats = {}

    def __init__(self):
        # Need to make sure fill_values list is instance attribute, not class attribute.
        # On read, this will be overwritten by the default in the ui.read (thus, in
        # the current implementation there can be no different default for different
        # Readers). On write, ui.py does not specify a default, so this line here matters.
        self.fill_values = copy.copy(self.fill_values)
        self.formats = copy.copy(self.formats)
        self.splitter = self.splitter_class()

    def process_lines(self, lines):
        """
        Strip out comment lines and blank lines from list of ``lines``

        Parameters
        ----------
        lines : list
            All lines in table

        Returns
        -------
        lines : list
            List of lines

        """
        nonblank_lines = (x for x in lines if x.strip())
        if self.comment:
            re_comment = re.compile(self.comment)
            return [x for x in nonblank_lines if not re_comment.match(x)]
        else:
            return [x for x in nonblank_lines]

    def get_data_lines(self, lines):
        """Set the ``data_lines`` attribute to the lines slice comprising the
        table data values."""
        data_lines = self.process_lines(lines)
        start_line = _get_line_index(self.start_line, data_lines)
        end_line = _get_line_index(self.end_line, data_lines)

        if start_line is not None or end_line is not None:
            self.data_lines = data_lines[slice(start_line, end_line)]
        else:  # Don't copy entire data lines unless necessary
            self.data_lines = data_lines

    def get_str_vals(self):
        """Return a generator that returns a list of column values (as strings)
        for each data line."""
        return self.splitter(self.data_lines)

    def masks(self, cols):
        """Set fill value for each column and then apply that fill value

        In the first step it is evaluated with value from ``fill_values`` applies to
        which column using ``fill_include_names`` and ``fill_exclude_names``.
        In the second step all replacements are done for the appropriate columns.
        """
        if self.fill_values:
            self._set_fill_values(cols)
            self._set_masks(cols)

    def _set_fill_values(self, cols):
        """Set the fill values of the individual cols based on fill_values of BaseData

        fill values has the following form:
        <fill_spec> = (<bad_value>, <fill_value>, <optional col_name>...)
        fill_values = <fill_spec> or list of <fill_spec>'s

        """
        if self.fill_values:
            # when we write tables the columns may be astropy.table.Columns
            # which don't carry a fill_values by default
            for col in cols:
                if ~hasattr(col, 'fill_values'):
                    col.fill_values = {}

            # if input is only one <fill_spec>, then make it a list
            with ignored(TypeError):
                self.fill_values[0] + ''
                self.fill_values = [self.fill_values]

            # Step 1: Set the default list of columns which are affected by
            # fill_values
            colnames = set(self.header.colnames)
            if self.fill_include_names is not None:
                colnames.intersection_update(self.fill_include_names)
            if self.fill_exclude_names is not None:
                colnames.difference_update(self.fill_exclude_names)

            # Step 2a: Find out which columns are affected by this tuple
            # iterate over reversed order, so last condition is set first and
            # overwritten by earlier conditions
            for replacement in reversed(self.fill_values):
                if len(replacement) < 2:
                    raise ValueError("Format of fill_values must be "
                                     "(<bad>, <fill>, <optional col1>, ...)")
                elif len(replacement) == 2:
                    affect_cols = colnames
                else:
                    affect_cols = replacement[2:]

                for i, key in ((i, x) for i, x in enumerate(self.header.colnames)
                               if x in affect_cols):
                    cols[i].fill_values[replacement[0]] = str(replacement[1])

    def _set_masks(self, cols):
        """Replace string values in col.str_vals and set masks"""
        if self.fill_values:
            for col in (col for col in cols if col.fill_values):
                col.mask = numpy.zeros(len(col.str_vals), dtype=numpy.bool)
                for i, str_val in ((i, x) for i, x in enumerate(col.str_vals)
                                   if x in col.fill_values):
                    col.str_vals[i] = col.fill_values[str_val]
                    col.mask[i] = True

    def _replace_vals(self, cols):
        """Replace string values in col.str_vals"""
        if self.fill_values:
            for col in (col for col in cols if col.fill_values):
                for i, str_val in ((i, x) for i, x in enumerate(col.str_vals)
                                   if x in col.fill_values):
                    col.str_vals[i] = col.fill_values[str_val]
                if masked in col.fill_values and hasattr(col, 'mask'):
                    mask_val = col.fill_values[masked]
                    for i in col.mask.nonzero()[0]:
                        col.str_vals[i] = mask_val

    def str_vals(self):
        '''convert all values in table to a list of lists of strings'''
        self._set_fill_values(self.cols)
        self._set_col_formats()
        for col in self.cols:
            col.str_vals = list(col_iter_str_vals(col))
        self._replace_vals(self.cols)
        return [col.str_vals for col in self.cols]

    def write(self, lines):
        if hasattr(self.start_line, '__call__'):
            raise TypeError('Start_line attribute cannot be callable for write()')
        else:
            data_start_line = self.start_line or 0

        while len(lines) < data_start_line:
            lines.append(itertools.cycle(self.write_spacer_lines))

        col_str_iters = self.str_vals()
        for vals in zip(*col_str_iters):
            lines.append(self.splitter.join(vals))

    def _set_col_formats(self):
        """
        """
        for col in self.cols:
            if col_getattr(col, 'name') in self.formats:
                col_setattr(col, 'format', self.formats[col.name])


def convert_numpy(numpy_type):
    """Return a tuple ``(converter_func, converter_type)``.  The converter
    function converts a list into a numpy array of the given ``numpy_type``.
    This type must be a valid `numpy type
    <http://docs.scipy.org/doc/numpy/user/basics.types.html>`_, e.g.
    numpy.int, numpy.uint, numpy.int8, numpy.int64, numpy.float, numpy.float64,
    numpy.str.  The converter type is used to track the generic data type (int,
    float, str) that is produced by the converter function.
    """

    # Infer converter type from an instance of numpy_type.
    type_name = numpy.array([], dtype=numpy_type).dtype.name
    if 'int' in type_name:
        converter_type = IntType
    elif 'float' in type_name:
        converter_type = FloatType
    elif 'bool' in type_name:
        converter_type = BoolType
    elif 'str' in type_name:
        converter_type = StrType
    else:
        converter_type = AllType

    def bool_converter(vals):
        """
        Convert values "False" and "True" to bools.  Raise an exception
        for any other string values.
        """
        # Try a smaller subset first for a long array
        if len(vals) > 10000:
            svals = numpy.asarray(vals[:1000])
            if not numpy.all((svals == 'False') | (svals == 'True')):
                raise ValueError('bool input strings must be only False or True')
        vals = numpy.asarray(vals)
        trues = vals == 'True'
        falses = vals == 'False'
        if not numpy.all(trues | falses):
            raise ValueError('bool input strings must be only False or True')
        return trues

    def generic_converter(vals):
        return numpy.array(vals, numpy_type)

    converter = bool_converter if converter_type is BoolType else generic_converter

    return converter, converter_type


class BaseOutputter(object):
    """Output table as a dict of column objects keyed on column name.  The
    table data are stored as plain python lists within the column objects.
    """
    converters = {}
    # Derived classes must define default_converters and __call__

    @staticmethod
    def _validate_and_copy(col, converters):
        """Validate the format for the type converters and then copy those
        which are valid converters for this column (i.e. converter type is
        a subclass of col.type)"""
        converters_out = []
        try:
            for converter in converters:
                converter_func, converter_type = converter
                if not issubclass(converter_type, NoType):
                    raise ValueError()
                if issubclass(converter_type, col.type):
                    converters_out.append((converter_func, converter_type))

        except (ValueError, TypeError):
            raise ValueError('Error: invalid format for converters, see documentation\n%s' %
                             converters)
        return converters_out

    def _convert_vals(self, cols):
        for col in cols:
            # If a specific dtype was specified for a column, then use that
            # to set the defaults, otherwise use the generic defaults.
            default_converters = ([convert_numpy(col.dtype)] if col.dtype
                                  else self.default_converters)

            # If the user supplied a specific convert then that takes precedence over defaults
            converters = self.converters.get(col.name, default_converters)

            col.converters = self._validate_and_copy(col, converters)

            while not hasattr(col, 'data'):
                try:
                    converter_func, converter_type = col.converters[0]
                    if not issubclass(converter_type, col.type):
                        raise TypeError()
                    col.data = converter_func(col.str_vals)
                    col.type = converter_type
                except (TypeError, ValueError):
                    col.converters.pop(0)
                except OverflowError:
                    # Overflow during conversion (most likely an int that doesn't fit in native C long).
                    # Put string at the top of the converters list for the next while iteration.
                    warnings.warn("OverflowError converting to {0} for column {1}, using string instead."
                                  .format(converter_type.__name__, col.name), AstropyWarning)
                    col.converters.insert(0, convert_numpy(numpy.str))
                except IndexError:
                    raise ValueError('Column %s failed to convert' % col.name)


class TableOutputter(BaseOutputter):
    """
    Output the table as an astropy.table.Table object.
    """

    default_converters = [convert_numpy(numpy.int),
                          convert_numpy(numpy.float),
                          convert_numpy(numpy.str)]

    def __call__(self, cols, meta):
        self._convert_vals(cols)

        # If there are any values that were filled and tagged with a mask bit then this
        # will be a masked table.  Otherwise use a plain table.
        masked = any(hasattr(col, 'mask') and numpy.any(col.mask) for col in cols)

        out = Table([x.data for x in cols], names=[x.name for x in cols], masked=masked,
                    meta=meta['table'])
        for col, out_col in zip(cols, out.columns.values()):
            if masked and hasattr(col, 'mask'):
                out_col.data.mask = col.mask
            for attr in ('format', 'unit', 'description'):
                if hasattr(col, attr):
                    setattr(out_col, attr, getattr(col, attr))
            if hasattr(col, 'meta'):
                out_col.meta.update(col.meta)

        return out


class MetaBaseReader(type):
    def __init__(cls, name, bases, dct):
        super(MetaBaseReader, cls).__init__(name, bases, dct)

        format = dct.get('_format_name')
        if format is None:
            return

        fast = dct.get('_fast')
        if fast is not None:
            FAST_CLASSES[format] = cls

        FORMAT_CLASSES[format] = cls

        io_formats = ['ascii.' + format] + dct.get('_io_registry_format_aliases', [])

        if dct.get('_io_registry_suffix'):
            func = functools.partial(connect.io_identify, dct['_io_registry_suffix'])
            connect.io_registry.register_identifier(io_formats[0], Table, func)

        for io_format in io_formats:
            func = functools.partial(connect.io_read, io_format)
            connect.io_registry.register_reader(io_format, Table, func)

            if dct.get('_io_registry_can_write', True):
                func = functools.partial(connect.io_write, io_format)
                connect.io_registry.register_writer(io_format, Table, func)


def _is_number(x):
    with ignored(ValueError):
        x = float(x)
        return True
    return False

def _apply_include_exclude_names(table, names, include_names, exclude_names):
    """
    Apply names, include_names and exclude_names to a table.

    Parameters
    ----------
    table : `~astropy.io.ascii.BaseReader`, array_like
        Input table
    names : list
        List of names to override those in table (set to None to use existing names)
    include_names : list
        List of names to include in output
    exclude_names: list
        List of names to exclude from output (applied after ``include_names``)

    """

    if names is not None:
        # Rename table column names to those passed by user
        # Temporarily rename with names that are not in `names` or `table.colnames`.
        # This ensures that rename succeeds regardless of existing names.
        xxxs = 'x' * max(len(name) for name in list(names) + list(table.colnames))
        for ii, colname in enumerate(table.colnames):
            table.rename_column(colname, xxxs + str(ii))

        for ii, name in enumerate(names):
            table.rename_column(xxxs + str(ii), name)

    names = set(table.colnames)
    if include_names is not None:<commit_msg>final revisions from @taldcroft's review<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
""" An extensible ASCII table reader and writer.

core.py:
  Core base classes and functions for reading and writing tables.

:Copyright: Smithsonian Astrophysical Observatory (2010)
:Author: <NAME> (<EMAIL>)
"""

from __future__ import absolute_import, division, print_function

import os
import re
import csv
import itertools
import functools
import numpy
import warnings
import copy

from ...extern import six
from ...extern.six.moves import zip
from ...extern.six.moves import cStringIO as StringIO
from ...utils.exceptions import AstropyWarning

from ...table import Table
from ...table.column import col_getattr, col_setattr, col_iter_str_vals
from ...utils.compat import ignored
from ...utils.data import get_readable_fileobj
from ...utils import OrderedDict
from . import connect

# Global dictionary mapping format arg to the corresponding Reader class
FORMAT_CLASSES = {}

# Similar dictionary for fast readers
FAST_CLASSES = {}

class MaskedConstant(numpy.ma.core.MaskedConstant):
    """A trivial extension of numpy.ma.masked

    We want to be able to put the generic term ``masked`` into a dictionary.
    In python 2.7 we can just use ``numpy.ma.masked``, but in python 3.1 and 3.2 that
    is not hashable, see https://github.com/numpy/numpy/issues/4660
    So, we need to extend it here with a hash value.
    """
    def __hash__(self):
        '''All instances of this class shall have the same hash.'''
        # Any large number will do.
        return 1234567890

masked = MaskedConstant()

class InconsistentTableError(ValueError):
    """
    Indicates that an input table is inconsistent in some way.

    The default behavior of ``BaseReader`` is to throw an instance of
    this class if a data row doesn't match the header.
    """

class OptionalTableImportError(ImportError):
    """
    Indicates that a dependency for table reading is not present.

    An instance of this class is raised whenever an optional reader
    with certain required dependencies cannot operate because of
    an ImportError.
    """

class ParameterError(NotImplementedError):
    """
    Indicates that a reader cannot handle a passed parameter.

    The C-based fast readers in ``io.ascii`` raise an instance of
    this error class upon encountering a parameter that the
    C engine cannot handle.
    """

class FastOptionsError(NotImplementedError):
    """
    Indicates that one of the specified options for fast
    reading is invalid.
    """

class NoType(object):
    """
    Superclass for ``StrType`` and ``NumType`` classes.

    This class is the default type of ``Column`` and provides a base
    class for other data types.
    """


class StrType(NoType):
    """
    Indicates that a column consists of text data.
    """


class NumType(NoType):
    """
    Indicates that a column consists of numerical data.
    """


class FloatType(NumType):
    """
    Describes floating-point data.
    """


class BoolType(NoType):
    """
    Describes boolean data.
    """


class IntType(NumType):
    """
    Describes integer data.
    """


class AllType(StrType, FloatType, IntType):
    """
    Subclass of all other data types.

    This type is returned by ``convert_numpy`` if the given numpy
    type does not match ``StrType``, ``FloatType``, or ``IntType``.
    """


class Column(object):
    """Table column.

    The key attributes of a Column object are:

    * **name** : column name
    * **type** : column type (NoType, StrType, NumType, FloatType, IntType)
    * **dtype** : numpy dtype (optional, overrides **type** if set)
    * **str_vals** : list of column values as strings
    * **data** : list of converted column values
    """
    def __init__(self, name):
        self.name = name
        self.type = NoType  # Generic type (Int, Float, Str etc)
        self.dtype = None  # Numpy dtype if available
        self.str_vals = []
        self.fill_values = {}


class BaseInputter(object):
    """
    Get the lines from the table input and return a list of lines.

    """
    def get_lines(self, table):
        """
        Get the lines from the ``table`` input. The input table can be one of:

        * File name
        * String (newline separated) with all header and data lines (must have at least 2 lines)
        * File-like object with read() method
        * List of strings

        Parameters
        ----------
        table : str, file_like, list
            Can be either a file name, string (newline separated) with all header and data
            lines (must have at least 2 lines), a file-like object with a ``read()`` method,
            or a list of strings.

        Returns
        -------
        lines : list
            List of lines
        """
        try:
            if (hasattr(table, 'read') or
                    ('\n' not in table + '' and '\r' not in table + '')):
                with get_readable_fileobj(table) as file_obj:
                    table = file_obj.read()
            lines = table.splitlines()
        except TypeError:
            try:
                # See if table supports indexing, slicing, and iteration
                table[0]
                table[0:1]
                iter(table)
                lines = table
            except TypeError:
                raise TypeError(
                    'Input "table" must be a string (filename or data) or an iterable')

        return self.process_lines(lines)

    def process_lines(self, lines):
        """Process lines for subsequent use.  In the default case do nothing.
        This routine is not generally intended for removing comment lines or
        stripping whitespace.  These are done (if needed) in the header and
        data line processing.

        Override this method if something more has to be done to convert raw
        input lines to the table rows.  For example the
        ContinuationLinesInputter derived class accounts for continuation
        characters if a row is split into lines."""
        return lines


class BaseSplitter(object):
    """
    Base splitter that uses python's split method to do the work.

    This does not handle quoted values.  A key feature is the formulation of
    __call__ as a generator that returns a list of the split line values at
    each iteration.

    There are two methods that are intended to be overridden, first
    ``process_line()`` to do pre-processing on each input line before splitting
    and ``process_val()`` to do post-processing on each split string value.  By
    default these apply the string ``strip()`` function.  These can be set to
    another function via the instance attribute or be disabled entirely, for
    example::

      reader.header.splitter.process_val = lambda x: x.lstrip()
      reader.data.splitter.process_val = None

    """

    #: one-character string used to separate fields
    delimiter = None

    def process_line(self, line):
        """Remove whitespace at the beginning or end of line.  This is especially useful for
        whitespace-delimited files to prevent spurious columns at the beginning or end."""
        return line.strip()

    def process_val(self, val):
        """Remove whitespace at the beginning or end of value."""
        return val.strip()

    def __call__(self, lines):
        if self.process_line:
            lines = (self.process_line(x) for x in lines)
        for line in lines:
            vals = line.split(self.delimiter)
            if self.process_val:
                yield [self.process_val(x) for x in vals]
            else:
                yield vals

    def join(self, vals):
        if self.delimiter is None:
            delimiter = ' '
        else:
            delimiter = self.delimiter
        return delimiter.join(str(x) for x in vals)


class DefaultSplitter(BaseSplitter):
    """Default class to split strings into columns using python csv.  The class
    attributes are taken from the csv Dialect class.

    Typical usage::

      # lines = ..
      splitter = ascii.DefaultSplitter()
      for col_vals in splitter(lines):
          for col_val in col_vals:
               ...

    """
    delimiter = ' '  #: one-character string used to separate fields.
    quotechar = '"'  #: control how instances of *quotechar* in a field are quoted
    doublequote = True  #: character to remove special meaning from following character
    escapechar = None  #: one-character stringto quote fields containing special characters
    quoting = csv.QUOTE_MINIMAL  #: control when quotes are recognised by the reader
    skipinitialspace = True  #: ignore whitespace immediately following the delimiter
    csv_writer = None
    csv_writer_out = StringIO()


    def process_line(self, line):
        """Remove whitespace at the beginning or end of line.  This is especially useful for
        whitespace-delimited files to prevent spurious columns at the beginning or end.
        If splitting on whitespace then replace unquoted tabs with space first"""
        if self.delimiter == '\s':
            line = _replace_tab_with_space(line, self.escapechar, self.quotechar)
        return line.strip()


    def __call__(self, lines):
        """Return an iterator over the table ``lines``, where each iterator output
        is a list of the split line values.

        Parameters
        ----------
        lines : list
            List of table lines

        Returns
        -------
        lines : iterator

        """
        if self.process_line:
            lines = [self.process_line(x) for x in lines]

        # In Python 2.x the inputs to csv cannot be unicode.  In Python 3 these
        # lines do nothing.
        escapechar = None if self.escapechar is None else str(self.escapechar)
        quotechar = None if self.quotechar is None else str(self.quotechar)
        delimiter = None if self.delimiter is None else str(self.delimiter)

        if delimiter == '\s':
            delimiter = ' '

        csv_reader = csv.reader(lines,
                                delimiter=delimiter,
                                doublequote=self.doublequote,
                                escapechar=escapechar,
                                quotechar=quotechar,
                                quoting=self.quoting,
                                skipinitialspace=self.skipinitialspace
                                )
        for vals in csv_reader:
            if self.process_val:
                yield [self.process_val(x) for x in vals]
            else:
                yield vals

    def join(self, vals):

        # In Python 2.x the inputs to csv cannot be unicode
        escapechar = None if self.escapechar is None else str(self.escapechar)
        quotechar = None if self.quotechar is None else str(self.quotechar)
        delimiter = ' ' if self.delimiter is None else str(self.delimiter)

        if self.csv_writer is None:
            self.csv_writer = csv.writer(self.csv_writer_out,
                                         delimiter=delimiter,
                                         doublequote=self.doublequote,
                                         escapechar=escapechar,
                                         quotechar=quotechar,
                                         quoting=self.quoting,
                                         lineterminator='',
                                         )
        self.csv_writer_out.seek(0)
        self.csv_writer_out.truncate()
        if self.process_val:
            vals = [self.process_val(x) for x in vals]
        self.csv_writer.writerow(vals)

        return self.csv_writer_out.getvalue()


def _replace_tab_with_space(line, escapechar, quotechar):
    """Replace tab with space within ``line`` while respecting quoted substrings"""
    newline = []
    in_quote = False
    lastchar = 'NONE'
    for char in line:
        if char == quotechar and lastchar != escapechar:
            in_quote = not in_quote
        if char == '\t' and not in_quote:
            char = ' '
        lastchar = char
        newline.append(char)
    return ''.join(newline)


def _get_line_index(line_or_func, lines):
    """Return the appropriate line index, depending on ``line_or_func`` which
    can be either a function, a positive or negative int, or None.
    """

    if hasattr(line_or_func, '__call__'):
        return line_or_func(lines)
    elif line_or_func:
        if line_or_func >= 0:
            return line_or_func
        else:
            n_lines = sum(1 for line in lines)
            return n_lines + line_or_func
    else:
        return line_or_func


class BaseHeader(object):
    """
    Base table header reader
    """
    auto_format = 'col%d'  #: format string for auto-generating column names
    start_line = None  #: None, int, or a function of ``lines`` that returns None or int
    comment = None  #: regular expression for comment lines
    splitter_class = DefaultSplitter  #: Splitter class for splitting data lines into columns
    names = None  #: list of names corresponding to each data column
    write_comment = False
    write_spacer_lines = ['ASCII_TABLE_WRITE_SPACER_LINE']

    def __init__(self):
        self.splitter = self.splitter_class()

    def _set_cols_from_names(self):
        self.cols = [Column(name=x) for x in self.names]

    def update_meta(self, lines, meta):
        """
        Extract any table-level metadata, e.g. keywords, comments, column metadata, from
        the table ``lines`` and update the OrderedDict ``meta`` in place.  This base
        method extracts comment lines and stores them in ``meta`` for output.
        """
        if self.comment:
            re_comment = re.compile(self.comment)
            comment_lines = [x for x in lines if re_comment.match(x)]
        else:
            comment_lines = []
        comment_lines = [re.sub('^' + self.comment, '', x).strip()
                         for x in comment_lines]
        if comment_lines:
            meta.setdefault('table', {})['comments'] = comment_lines

    def get_cols(self, lines):
        """Initialize the header Column objects from the table ``lines``.

        Based on the previously set Header attributes find or create the column names.
        Sets ``self.cols`` with the list of Columns.

        Parameters
        ----------
        lines : list
            List of table lines

        """

        start_line = _get_line_index(self.start_line, self.process_lines(lines))
        if start_line is None:
            # No header line so auto-generate names from n_data_cols
            # Get the data values from the first line of table data to determine n_data_cols
            try:
                first_data_vals = next(self.data.get_str_vals())
            except StopIteration:
                raise InconsistentTableError('No data lines found so cannot autogenerate '
                                             'column names')
            n_data_cols = len(first_data_vals)
            self.names = [self.auto_format % i for i in range(1, n_data_cols + 1)]

        else:
            for i, line in enumerate(self.process_lines(lines)):
                if i == start_line:
                    break
            else:  # No header line matching
                raise ValueError('No header line found in table')

            self.names = next(self.splitter([line]))

        self._set_cols_from_names()

    def process_lines(self, lines):
        """Generator to yield non-comment lines"""
        if self.comment:
            re_comment = re.compile(self.comment)
        # Yield non-comment lines
        for line in lines:
            if line and (not self.comment or not re_comment.match(line)):
                yield line

    def write_comments(self, lines, meta):
        if self.write_comment is not False:
            for comment in meta.get('comments', []):
                lines.append(self.write_comment + comment)

    def write(self, lines):
        if self.start_line is not None:
            for i, spacer_line in zip(range(self.start_line),
                                      itertools.cycle(self.write_spacer_lines)):
                lines.append(spacer_line)
            lines.append(self.splitter.join([col_getattr(x, 'name') for x in self.cols]))

    @property
    def colnames(self):
        """Return the column names of the table"""
        return tuple(col.name if isinstance(col, Column) else col_getattr(col, 'name')
                     for col in self.cols)

    def get_type_map_key(self, col):
        return col.raw_type

    def get_col_type(self, col):
        try:
            type_map_key = self.get_type_map_key(col)
            return self.col_type_map[type_map_key.lower()]
        except KeyError:
            raise ValueError('Unknown data type ""%s"" for column "%s"' % (
                col.raw_type, col.name))

    def check_column_names(self, names, strict_names, guessing):
        """
        Check column names.

        This must be done before applying the names transformation
        so that guessing will fail appropriately if ``names`` is supplied.
        For instance if the basic reader is given a table with no column header
        row.

        Parameters
        ----------
        names : list
            User-supplied list of column names
        strict_names : bool
            Whether to impose extra requirements on names
        guessing : bool
            True if this method is being called while guessing the table format
        """
        if strict_names:
            # Impose strict requirements on column names (normally used in guessing)
            bads = [" ", ",", "|", "\t", "'", '"']
            for name in self.colnames:
                if (_is_number(name) or
                    len(name) == 0 or
                    name[0] in bads or
                    name[-1] in bads):
                    raise ValueError('Column name {0!r} does not meet strict name requirements'
                                     .format(name))
        # When guessing require at least two columns
        if guessing and len(self.colnames) <= 1:
            raise ValueError

        if names is not None and len(names) != len(self.colnames):
            raise ValueError('Length of names argument ({0}) does not match number'
                             ' of table columns ({1})'.format(len(names), len(self.colnames)))


class BaseData(object):
    """
    Base table data reader.
    """
    start_line = None  #: None, int, or a function of ``lines`` that returns None or int
    end_line = None  #: None, int, or a function of ``lines`` that returns None or int
    comment = None  #: Regular expression for comment lines
    splitter_class = DefaultSplitter  #: Splitter class for splitting data lines into columns
    write_spacer_lines = ['ASCII_TABLE_WRITE_SPACER_LINE']
    fill_include_names = None
    fill_exclude_names = None
    # Currently, the default matches the numpy default for masked values.
    fill_values = [(masked, '--')]
    formats = {}

    def __init__(self):
        # Need to make sure fill_values list is instance attribute, not class attribute.
        # On read, this will be overwritten by the default in the ui.read (thus, in
        # the current implementation there can be no different default for different
        # Readers). On write, ui.py does not specify a default, so this line here matters.
        self.fill_values = copy.copy(self.fill_values)
        self.formats = copy.copy(self.formats)
        self.splitter = self.splitter_class()

    def process_lines(self, lines):
        """
        Strip out comment lines and blank lines from list of ``lines``

        Parameters
        ----------
        lines : list
            All lines in table

        Returns
        -------
        lines : list
            List of lines

        """
        nonblank_lines = (x for x in lines if x.strip())
        if self.comment:
            re_comment = re.compile(self.comment)
            return [x for x in nonblank_lines if not re_comment.match(x)]
        else:
            return [x for x in nonblank_lines]

    def get_data_lines(self, lines):
        """Set the ``data_lines`` attribute to the lines slice comprising the
        table data values."""
        data_lines = self.process_lines(lines)
        start_line = _get_line_index(self.start_line, data_lines)
        end_line = _get_line_index(self.end_line, data_lines)

        if start_line is not None or end_line is not None:
            self.data_lines = data_lines[slice(start_line, end_line)]
        else:  # Don't copy entire data lines unless necessary
            self.data_lines = data_lines

    def get_str_vals(self):
        """Return a generator that returns a list of column values (as strings)
        for each data line."""
        return self.splitter(self.data_lines)

    def masks(self, cols):
        """Set fill value for each column and then apply that fill value

        In the first step it is evaluated with value from ``fill_values`` applies to
        which column using ``fill_include_names`` and ``fill_exclude_names``.
        In the second step all replacements are done for the appropriate columns.
        """
        if self.fill_values:
            self._set_fill_values(cols)
            self._set_masks(cols)

    def _set_fill_values(self, cols):
        """Set the fill values of the individual cols based on fill_values of BaseData

        fill values has the following form:
        <fill_spec> = (<bad_value>, <fill_value>, <optional col_name>...)
        fill_values = <fill_spec> or list of <fill_spec>'s

        """
        if self.fill_values:
            # when we write tables the columns may be astropy.table.Columns
            # which don't carry a fill_values by default
            for col in cols:
                if ~hasattr(col, 'fill_values'):
                    col.fill_values = {}

            # if input is only one <fill_spec>, then make it a list
            with ignored(TypeError):
                self.fill_values[0] + ''
                self.fill_values = [self.fill_values]

            # Step 1: Set the default list of columns which are affected by
            # fill_values
            colnames = set(self.header.colnames)
            if self.fill_include_names is not None:
                colnames.intersection_update(self.fill_include_names)
            if self.fill_exclude_names is not None:
                colnames.difference_update(self.fill_exclude_names)

            # Step 2a: Find out which columns are affected by this tuple
            # iterate over reversed order, so last condition is set first and
            # overwritten by earlier conditions
            for replacement in reversed(self.fill_values):
                if len(replacement) < 2:
                    raise ValueError("Format of fill_values must be "
                                     "(<bad>, <fill>, <optional col1>, ...)")
                elif len(replacement) == 2:
                    affect_cols = colnames
                else:
                    affect_cols = replacement[2:]

                for i, key in ((i, x) for i, x in enumerate(self.header.colnames)
                               if x in affect_cols):
                    cols[i].fill_values[replacement[0]] = str(replacement[1])

    def _set_masks(self, cols):
        """Replace string values in col.str_vals and set masks"""
        if self.fill_values:
            for col in (col for col in cols if col.fill_values):
                col.mask = numpy.zeros(len(col.str_vals), dtype=numpy.bool)
                for i, str_val in ((i, x) for i, x in enumerate(col.str_vals)
                                   if x in col.fill_values):
                    col.str_vals[i] = col.fill_values[str_val]
                    col.mask[i] = True

    def _replace_vals(self, cols):
        """Replace string values in col.str_vals"""
        if self.fill_values:
            for col in (col for col in cols if col.fill_values):
                for i, str_val in ((i, x) for i, x in enumerate(col.str_vals)
                                   if x in col.fill_values):
                    col.str_vals[i] = col.fill_values[str_val]
                if masked in col.fill_values and hasattr(col, 'mask'):
                    mask_val = col.fill_values[masked]
                    for i in col.mask.nonzero()[0]:
                        col.str_vals[i] = mask_val

    def str_vals(self):
        '''convert all values in table to a list of lists of strings'''
        self._set_fill_values(self.cols)
        self._set_col_formats()
        for col in self.cols:
            col.str_vals = list(col_iter_str_vals(col))
        self._replace_vals(self.cols)
        return [col.str_vals for col in self.cols]

    def write(self, lines):
        if hasattr(self.start_line, '__call__'):
            raise TypeError('Start_line attribute cannot be callable for write()')
        else:
            data_start_line = self.start_line or 0

        while len(lines) < data_start_line:
            lines.append(itertools.cycle(self.write_spacer_lines))

        col_str_iters = self.str_vals()
        for vals in zip(*col_str_iters):
            lines.append(self.splitter.join(vals))

    def _set_col_formats(self):
        """
        """
        for col in self.cols:
            if col_getattr(col, 'name') in self.formats:
                col_setattr(col, 'format', self.formats[col.name])


def convert_numpy(numpy_type):
    """Return a tuple ``(converter_func, converter_type)``.  The converter
    function converts a list into a numpy array of the given ``numpy_type``.
    This type must be a valid `numpy type
    <http://docs.scipy.org/doc/numpy/user/basics.types.html>`_, e.g.
    numpy.int, numpy.uint, numpy.int8, numpy.int64, numpy.float, numpy.float64,
    numpy.str.  The converter type is used to track the generic data type (int,
    float, str) that is produced by the converter function.
    """

    # Infer converter type from an instance of numpy_type.
    type_name = numpy.array([], dtype=numpy_type).dtype.name
    if 'int' in type_name:
        converter_type = IntType
    elif 'float' in type_name:
        converter_type = FloatType
    elif 'bool' in type_name:
        converter_type = BoolType
    elif 'str' in type_name:
        converter_type = StrType
    else:
        converter_type = AllType

    def bool_converter(vals):
        """
        Convert values "False" and "True" to bools.  Raise an exception
        for any other string values.
        """
        # Try a smaller subset first for a long array
        if len(vals) > 10000:
            svals = numpy.asarray(vals[:1000])
            if not numpy.all((svals == 'False') | (svals == 'True')):
                raise ValueError('bool input strings must be only False or True')
        vals = numpy.asarray(vals)
        trues = vals == 'True'
        falses = vals == 'False'
        if not numpy.all(trues | falses):
            raise ValueError('bool input strings must be only False or True')
        return trues

    def generic_converter(vals):
        return numpy.array(vals, numpy_type)

    converter = bool_converter if converter_type is BoolType else generic_converter

    return converter, converter_type


class BaseOutputter(object):
    """Output table as a dict of column objects keyed on column name.  The
    table data are stored as plain python lists within the column objects.
    """
    converters = {}
    # Derived classes must define default_converters and __call__

    @staticmethod
    def _validate_and_copy(col, converters):
        """Validate the format for the type converters and then copy those
        which are valid converters for this column (i.e. converter type is
        a subclass of col.type)"""
        converters_out = []
        try:
            for converter in converters:
                converter_func, converter_type = converter
                if not issubclass(converter_type, NoType):
                    raise ValueError()
                if issubclass(converter_type, col.type):
                    converters_out.append((converter_func, converter_type))

        except (ValueError, TypeError):
            raise ValueError('Error: invalid format for converters, see documentation\n%s' %
                             converters)
        return converters_out

    def _convert_vals(self, cols):
        for col in cols:
            # If a specific dtype was specified for a column, then use that
            # to set the defaults, otherwise use the generic defaults.
            default_converters = ([convert_numpy(col.dtype)] if col.dtype
                                  else self.default_converters)

            # If the user supplied a specific convert then that takes precedence over defaults
            converters = self.converters.get(col.name, default_converters)

            col.converters = self._validate_and_copy(col, converters)

            while not hasattr(col, 'data'):
                try:
                    converter_func, converter_type = col.converters[0]
                    if not issubclass(converter_type, col.type):
                        raise TypeError()
                    col.data = converter_func(col.str_vals)
                    col.type = converter_type
                except (TypeError, ValueError):
                    col.converters.pop(0)
                except OverflowError:
                    # Overflow during conversion (most likely an int that doesn't fit in native C long).
                    # Put string at the top of the converters list for the next while iteration.
                    warnings.warn("OverflowError converting to {0} for column {1}, using string instead."
                                  .format(converter_type.__name__, col.name), AstropyWarning)
                    col.converters.insert(0, convert_numpy(numpy.str))
                except IndexError:
                    raise ValueError('Column %s failed to convert' % col.name)


class TableOutputter(BaseOutputter):
    """
    Output the table as an astropy.table.Table object.
    """

    default_converters = [convert_numpy(numpy.int),
                          convert_numpy(numpy.float),
                          convert_numpy(numpy.str)]

    def __call__(self, cols, meta):
        self._convert_vals(cols)

        # If there are any values that were filled and tagged with a mask bit then this
        # will be a masked table.  Otherwise use a plain table.
        masked = any(hasattr(col, 'mask') and numpy.any(col.mask) for col in cols)

        out = Table([x.data for x in cols], names=[x.name for x in cols], masked=masked,
                    meta=meta['table'])
        for col, out_col in zip(cols, out.columns.values()):
            if masked and hasattr(col, 'mask'):
                out_col.data.mask = col.mask
            for attr in ('format', 'unit', 'description'):
                if hasattr(col, attr):
                    setattr(out_col, attr, getattr(col, attr))
            if hasattr(col, 'meta'):
                out_col.meta.update(col.meta)

        return out


class MetaBaseReader(type):
    def __init__(cls, name, bases, dct):
        super(MetaBaseReader, cls).__init__(name, bases, dct)

        format = dct.get('_format_name')
        if format is None:
            return

        fast = dct.get('_fast')
        if fast is not None:
            FAST_CLASSES[format] = cls

        FORMAT_CLASSES[format] = cls

        io_formats = ['ascii.' + format] + dct.get('_io_registry_format_aliases', [])

        if dct.get('_io_registry_suffix'):
            func = functools.partial(connect.io_identify, dct['_io_registry_suffix'])
            connect.io_registry.register_identifier(io_formats[0], Table, func)

        for io_format in io_formats:
            func = functools.partial(connect.io_read, io_format)
            connect.io_registry.register_reader(io_format, Table, func)

            if dct.get('_io_registry_can_write', True):
                func = functools.partial(connect.io_write, io_format)
                connect.io_registry.register_writer(io_format, Table, func)


def _is_number(x):
    with ignored(ValueError):
        x = float(x)
        return True
    return False

def _apply_include_exclude_names(table, names, include_names, exclude_names):
    """
    Apply names, include_names and exclude_names to a table.

    Parameters
    ----------
    table : `~astropy.table.Table`
        Input table
    names : list
        List of names to override those in table (set to None to use existing names)
    include_names : list
        List of names to include in output
    exclude_names: list
        List of names to exclude from output (applied after ``include_names``)

    """

    if names is not None:
        # Rename table column names to those passed by user
        # Temporarily rename with names that are not in `names` or `table.colnames`.
        # This ensures that rename succeeds regardless of existing names.
        xxxs = 'x' * max(len(name) for name in list(names) + list(table.colnames))
        for ii, colname in enumerate(table.colnames):
            table.rename_column(colname, xxxs + str(ii))

        for ii, name in enumerate(names):
            table.rename_column(xxxs + str(ii), name)

    names = set(table.colnames)
    if include_names is not None:
{'fraction_non_alphanumeric': 0.08352848172068672, 'fraction_numerical': 0.002267201646704354, 'mean_word_length': 2.0673925973372373, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 33, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 50, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3708795', 'n_tokens_mistral': 18448, 'n_tokens_neox': 17670, 'n_words': 6321}