<commit_before>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

"""
Interface and base implmentations for a generic table type

Interface Table:
  getColumns(): returns a sequence of objects that represent columns
  getRows():    returns a sequence of objects that represent rows
  getValue(row, column): returns an object representing a value given
                         two objects from getRows()/getColumns()

see tableoutput.py for useful methods for rendering tables in different ways
"""

from __future__ import unicode_literals, print_function, absolute_import

from amcat.tools import toolkit, idlabel
from amcat.tools.toolkit import isnull
from amcat.tools.table import tableoutput

from collections import namedtuple

import types, re
from amcat.contrib.oset import OrderedSet

import logging; log = logging.getLogger(__name__)


def trivialCellFunc(row, col):
    """'Default' cell function that returns a string representation row/col"""
    return "%s/%s" % (row, col)

class Table(object):
    """Generic interface on rectangular tables.

    Supports read access using getRows/getColumns/getValue and
    using index access to and iteration over the 'Named' rows
    e.g. print row[1].colA; for row in table: print(row.colB)

    This is primarily meant for subclassing, but a base implementation
    works by giving columns and rows as sequences and a function that
    maps column and row to a value.
    """
    def __init__(self, columns=None, rows = None, cellfunc = trivialCellFunc,
                 rowNamesRequired = False):
        """
        @param columns: a sequence of columns, or None if getColumns is overridden
        @param rows:a sequence of rows, or None if getRows is overridden
        @param cellfunc: a function taking a row and column argument, or None
                         if getValue is overridden.
        @param rowNamesRequired: a hint to output functions that rownames should be printed
        """
        self.columns    = isnull(columns, []) 
        self.rows       = isnull(rows, [])
        self.cellfunc   = cellfunc
        self.rowNamesRequired = rowNamesRequired

    # Basic table interface
    def getValue(self, row, column):
        """Get the value corresponding to this row and column"""
        result =  self.cellfunc(row, column)
        return result
    def getRows(self):
        """Get a sequence of objects representing the rows"""
        return self.rows
    def getColumns(self):
        """Get a sequence of objects representing the columns"""
        return self.columns

    # Convenience access using iteration / index and NamedRows
    def getNamedRows(self):
        """Get a sequence of NamedRow objects that can be used to access the values"""
        for r in self.getRows():
            yield NamedRow(self, r)
    def getNamedRow(self, rowname):
        """Get a NamedRow object for the given row(name)"""
        for r in self.getRows():
            if r == rowname:
                return NamedRow(self, r)
        return None       
    def __iter__(self):
        return iter(self.getNamedRows())
    def __getitem__(self, index):
        """Get the n-th column"""
        return list(self.getRows())[index]


    def getColumnByLabel(self, label):
        """Return the column that matches the given label, or None"""
        stringify = unicode if type(label) == unicode else str
        for c in self.getColumns():
            if stringify(c) == label: return c

    def output(self, **kargs):
        """Output the table; see tableoutput.table2unicode for options"""
        return tableoutput.table2unicode(self, **kargs)

    def to_list(self, tuple_name="row"):
        """Return the data in the table as a sequence of named tuples

        @param tuple_name: the name for the named tuples, or None to get simple tuples
        """
        cols = self.getColumns()
        if tuple_name:
            colnames = [re.sub(r"\W", "", str(col)) for col in cols]
            t = namedtuple(tuple_name, colnames, rename=True)
            factory = lambda values: t(*values)
        else:
            factory = tuple
        for row in self.getRows():
            yield factory([self.getValue(row, col) for col in cols])
            

            
    
class NamedRow(object):
    """Interface for a row in a table that supports attribute and index access"""
    def __init__(self, table, row):
        self.table = table
        self.row = row
    def get(self, column):
        """Get the specified column on this row?"""
        for col in self.table.getColumns():
            if str(col) == column:
                return self.table.getValue(self.row, col)
    def __getattr__(self, attr):
        """If attr is the str(col) for any column, return that column"""
        if attr != 'table':
            for col in self.table.getColumns():
                if str(col) == attr:
                    return self.table.getValue(self.row, col)
        return super(NamedRow, self).__getattribute__(attr)
    def __getitem__(self, index):
        """Get the n-th column"""
        col = list(self.table.getColumns())[index]
        return self.table.getValue(self.row, col)
    def __iter__(self):
        """Iterate over the columns"""
        for c in self.table.getColumns():
            yield self.table.getValue( self.row, c)

    
# Why are these here?
OBJECTCOLUMN_PROPERTIES = ('fieldname', 'fieldtype', 'label',
                           'visible', 'editable', 'url')

class ObjectTable(Table):
    """
    Convenience subclass of Table that assumes the rows contain
    a domain object and the columns are properties of those objects
    The colunms should be ObjectColunms or some other object
    that has a getCell(row) -> value function
    """
    def __init__(self, rows = None, columns = None):
        Table.__init__(self, columns=[], rows = rows or [])
        if columns:
            for column in columns:
                self.addColumn(column)
    def addColumn(self, col, label=None, **kargs):
        """Add column to Table3 object
        
        @type col: ObjectColumn, string, or (lambda-)function
        @param col: Column you want to add to this table. If it is
        a function, it will be called with an object every time
        a cell is created. The returned data will be used to fill it. If
        if is a string, use it as an attribute getter 
        
        @type label: str or unicode
        @param label: String for user-friendly column-identification.
        If not provided, the __name__ attribute of `col` will be used.
        
        @type return: NoneType 
        """
        if hasattr(col, '__call__'): # function
            if label is None: label = col.__name__
            if label == '<lambda>': label = ''
            
            col = ObjectColumn(label, col, **kargs)
        elif type(col) in (str, unicode):
            col = AttributeColumn(col, label, **kargs)
        self.columns.append(col)
        return col
    def getValue(self, row, column):
        """Get the column-value for the given row object"""
        return column.getCell(row)
        
class ObjectColumn(object):
    """An ObjectColumn is a column on an ObjectTable that gets a value
    on an object by calling the cellfunc(obj)"""
    def __init__(self, label, cellfunc=None, fieldname=None, fieldtype=None, 
                 visible=True, editable=True, url=None):
        self.label = label
        self.cellfunc = cellfunc
        self.fieldname = fieldname or label<commit_msg>add option to insert column at location<commit_after>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

"""
Interface and base implmentations for a generic table type

Interface Table:
  getColumns(): returns a sequence of objects that represent columns
  getRows():    returns a sequence of objects that represent rows
  getValue(row, column): returns an object representing a value given
                         two objects from getRows()/getColumns()

see tableoutput.py for useful methods for rendering tables in different ways
"""

from __future__ import unicode_literals, print_function, absolute_import

from amcat.tools import toolkit, idlabel
from amcat.tools.toolkit import isnull
from amcat.tools.table import tableoutput

from collections import namedtuple

import types, re
from amcat.contrib.oset import OrderedSet

import logging; log = logging.getLogger(__name__)


def trivialCellFunc(row, col):
    """'Default' cell function that returns a string representation row/col"""
    return "%s/%s" % (row, col)

class Table(object):
    """Generic interface on rectangular tables.

    Supports read access using getRows/getColumns/getValue and
    using index access to and iteration over the 'Named' rows
    e.g. print row[1].colA; for row in table: print(row.colB)

    This is primarily meant for subclassing, but a base implementation
    works by giving columns and rows as sequences and a function that
    maps column and row to a value.
    """
    def __init__(self, columns=None, rows = None, cellfunc = trivialCellFunc,
                 rowNamesRequired = False):
        """
        @param columns: a sequence of columns, or None if getColumns is overridden
        @param rows:a sequence of rows, or None if getRows is overridden
        @param cellfunc: a function taking a row and column argument, or None
                         if getValue is overridden.
        @param rowNamesRequired: a hint to output functions that rownames should be printed
        """
        self.columns    = isnull(columns, []) 
        self.rows       = isnull(rows, [])
        self.cellfunc   = cellfunc
        self.rowNamesRequired = rowNamesRequired

    # Basic table interface
    def getValue(self, row, column):
        """Get the value corresponding to this row and column"""
        result =  self.cellfunc(row, column)
        return result
    def getRows(self):
        """Get a sequence of objects representing the rows"""
        return self.rows
    def getColumns(self):
        """Get a sequence of objects representing the columns"""
        return self.columns

    # Convenience access using iteration / index and NamedRows
    def getNamedRows(self):
        """Get a sequence of NamedRow objects that can be used to access the values"""
        for r in self.getRows():
            yield NamedRow(self, r)
    def getNamedRow(self, rowname):
        """Get a NamedRow object for the given row(name)"""
        for r in self.getRows():
            if r == rowname:
                return NamedRow(self, r)
        return None       
    def __iter__(self):
        return iter(self.getNamedRows())
    def __getitem__(self, index):
        """Get the n-th column"""
        return list(self.getRows())[index]


    def getColumnByLabel(self, label):
        """Return the column that matches the given label, or None"""
        stringify = unicode if type(label) == unicode else str
        for c in self.getColumns():
            if stringify(c) == label: return c

    def output(self, **kargs):
        """Output the table; see tableoutput.table2unicode for options"""
        return tableoutput.table2unicode(self, **kargs)

    def to_list(self, tuple_name="row"):
        """Return the data in the table as a sequence of named tuples

        @param tuple_name: the name for the named tuples, or None to get simple tuples
        """
        cols = self.getColumns()
        if tuple_name:
            colnames = [re.sub(r"\W", "", str(col)) for col in cols]
            t = namedtuple(tuple_name, colnames, rename=True)
            factory = lambda values: t(*values)
        else:
            factory = tuple
        for row in self.getRows():
            yield factory([self.getValue(row, col) for col in cols])
            

            
    
class NamedRow(object):
    """Interface for a row in a table that supports attribute and index access"""
    def __init__(self, table, row):
        self.table = table
        self.row = row
    def get(self, column):
        """Get the specified column on this row?"""
        for col in self.table.getColumns():
            if str(col) == column:
                return self.table.getValue(self.row, col)
    def __getattr__(self, attr):
        """If attr is the str(col) for any column, return that column"""
        if attr != 'table':
            for col in self.table.getColumns():
                if str(col) == attr:
                    return self.table.getValue(self.row, col)
        return super(NamedRow, self).__getattribute__(attr)
    def __getitem__(self, index):
        """Get the n-th column"""
        col = list(self.table.getColumns())[index]
        return self.table.getValue(self.row, col)
    def __iter__(self):
        """Iterate over the columns"""
        for c in self.table.getColumns():
            yield self.table.getValue( self.row, c)

    
# Why are these here?
OBJECTCOLUMN_PROPERTIES = ('fieldname', 'fieldtype', 'label',
                           'visible', 'editable', 'url')

class ObjectTable(Table):
    """
    Convenience subclass of Table that assumes the rows contain
    a domain object and the columns are properties of those objects
    The colunms should be ObjectColunms or some other object
    that has a getCell(row) -> value function
    """
    def __init__(self, rows = None, columns = None):
        Table.__init__(self, columns=[], rows = rows or [])
        if columns:
            for column in columns:
                self.addColumn(column)
    def addColumn(self, col, label=None, index=None, **kargs):
        """Add column to Table3 object
        
        @type col: ObjectColumn, string, or (lambda-)function
        @param col: Column you want to add to this table. If it is
        a function, it will be called with an object every time
        a cell is created. The returned data will be used to fill it. If
        if is a string, use it as an attribute getter 
        
        @type label: str or unicode
        @param label: String for user-friendly column-identification.
        If not provided, the __name__ attribute of `col` will be used.
        
        @type return: NoneType 
        """
        if hasattr(col, '__call__'): # function
            if label is None: label = col.__name__
            if label == '<lambda>': label = ''
            
            col = ObjectColumn(label, col, **kargs)
        elif type(col) in (str, unicode):
            col = AttributeColumn(col, label, **kargs)
	if index is not None:
	    self.columns.insert(index, col)
	else:
	    self.columns.append(col)
        return col
    def getValue(self, row, column):
        """Get the column-value for the given row object"""
        return column.getCell(row)
        
class ObjectColumn(object):
    """An ObjectColumn is a column on an ObjectTable that gets a value
    on an object by calling the cellfunc(obj)"""
    def __init__(self, label, cellfunc=None, fieldname=None, fieldtype=None, 
                 visible=True, editable=True, url=None):
        self.label = label
        self.cellfunc = cellfunc
        self.fieldname = fieldname or label
{'fraction_non_alphanumeric': 0.09951620162016202, 'fraction_numerical': 0.0005625562556255626, 'mean_word_length': 2.1961524631427545, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7448699', 'n_tokens_mistral': 4495, 'n_tokens_neox': 4400, 'n_words': 1812}