<commit_before>"""
******
Tables
******

Table entity


syn.getColumns

syn.getTableColumns

"""
from synapseclient.exceptions import *
from synapseclient.dict_object import DictObject
from synapseclient.utils import id_of, query_limit_and_offset

#COLUMN_TYPES = ['STRING', 'DOUBLE', 'LONG', 'BOOLEAN', 'DATE', 'FILEHANDLEID']
DTYPE_2_TABLETYPE = {'?':'BOOLEAN',
                     'd': 'DOUBLE', 'g': 'DOUBLE', 'e': 'DOUBLE', 'f': 'DOUBLE',
                     'b': 'LONG', 'B': 'LONG', 'h': 'LONG', 'H': 'LONG',
                     'i': 'LONG', 'I': 'LONG', 'l': 'LONG', 'L': 'LONG',
                     'm': 'LONG', 'q': 'LONG', 'Q': 'LONG',
                     'S': 'STRING', 'U': 'STRING', 'O': 'STRING'}


def df2Table(df, tableName, parentProject):
    """Creates a new table from data in pandas data frame.
    parameters: df, tableName, parentProject
    """

    #Create columns:
    cols = list()
    for col in df:
        columnType = DTYPE_2_TABLETYPE[df[col].dtype.char]
        if columnType == 'STRING':
            size = min(1000, max(30, df[col].str.len().max()*1.5))  #Determine lenght of longest string
            cols.append(ColumnModel(name=col, columnType=columnType, maximumSize=size, defaultValue=''))
        else:
            cols.append(ColumnModel(name=col, columnType=columnType))
    cols = [syn.store(col) for col in cols]

    #Create Table
    table1 = Table(name=tableName, columns=cols, parent=parentProject)
    table1 = syn.store(table1)


    #Add data to Table
    for i in range(0, df.shape[0]/250+1):
        start =  i*250
        end = min((i+1)*250, df.shape[0])
        print start, end
        rowset1 = RowSet(columns=cols, table=table1,
                         rows=[Row(list(df.ix[j,:])) for j in range(start,end)])
        rowset1 = syn.store(rowset1)

    return table1


class ColumnModel(DictObject):
    """
    """

    @classmethod
    def getURI(cls, id):
        return '/column/%s' % id


    def __init__(self, **kwargs):
        super(ColumnModel, self).__init__(kwargs)

    def postURI(self):
        return '/column'


# RowSet
# org.sagebionetworks.repo.model.table.RowSet
# headers       ARRAY<STRING>  The list of ColumnModels ID that describes the rows of this set.
# etag          STRING         Any RowSet returned from Synapse will contain the current etag of the change set. To update any rows from a RowSet the etag must be provided with the POST.<commit_msg>Adds missing synapse object for tables.<commit_after>"""
******
Tables
******

Table entity


syn.getColumns

syn.getTableColumns

"""
from synapseclient.exceptions import *
from synapseclient.dict_object import DictObject
from synapseclient.utils import id_of, query_limit_and_offset
from synapseclient.entity import Table

#COLUMN_TYPES = ['STRING', 'DOUBLE', 'LONG', 'BOOLEAN', 'DATE', 'FILEHANDLEID']
DTYPE_2_TABLETYPE = {'?':'BOOLEAN',
                     'd': 'DOUBLE', 'g': 'DOUBLE', 'e': 'DOUBLE', 'f': 'DOUBLE',
                     'b': 'LONG', 'B': 'LONG', 'h': 'LONG', 'H': 'LONG',
                     'i': 'LONG', 'I': 'LONG', 'l': 'LONG', 'L': 'LONG',
                     'm': 'LONG', 'q': 'LONG', 'Q': 'LONG',
                     'S': 'STRING', 'U': 'STRING', 'O': 'STRING'}


def df2Table(df, syn,  tableName, parentProject):
    """Creates a new table from data in pandas data frame.
    parameters: df, tableName, parentProject
    """

    #Create columns:
    print df.shape
    cols = list()
    for col in df:
        columnType = DTYPE_2_TABLETYPE[df[col].dtype.char]
        if columnType == 'STRING':
            size = min(1000, max(30, df[col].str.len().max()*1.5))  #Determine lenght of longest string
            cols.append(ColumnModel(name=col, columnType=columnType, maximumSize=size, defaultValue=''))
        else:
            cols.append(ColumnModel(name=col, columnType=columnType))
    cols = [syn.store(col) for col in cols]

    #Create Table
    table1 = Table(name=tableName, columns=cols, parent=parentProject)
    table1 = syn.store(table1)


    #Add data to Table
    for i in range(0, df.shape[0]/1200+1):
        start =  i*1200
        end = min((i+1)*1200, df.shape[0])
        print start, end
        rowset1 = RowSet(columns=cols, table=table1,
                         rows=[Row(list(df.ix[j,:])) for j in range(start,end)])
        #print len(rowset1.rows)
        rowset1 = syn.store(rowset1)

    return table1


class ColumnModel(DictObject):
    """
    """

    @classmethod
    def getURI(cls, id):
        return '/column/%s' % id


    def __init__(self, **kwargs):
        super(ColumnModel, self).__init__(kwargs)

    def postURI(self):
        return '/column'


# RowSet
# org.sagebionetworks.repo.model.table.RowSet
# headers       ARRAY<STRING>  The list of ColumnModels ID that describes the rows of this set.
# etag          STRING         Any RowSet returned from Synapse will contain the current etag of the change set. To update any rows from a RowSet the etag must be provided with the POST.
{'fraction_non_alphanumeric': 0.14599122457120065, 'fraction_numerical': 0.013960909453530115, 'mean_word_length': 2.75374251497006, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2846148', 'n_tokens_mistral': 1652, 'n_tokens_neox': 1584, 'n_words': 449}