<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from backports import csv
import io
import filecmp
import math
import os
import random
import sys
import tempfile
import time
import uuid
import six
from builtins import zip
from nose.tools import assert_equals, assert_less, assert_not_equal, assert_dict_equal, assert_false, assert_true
from datetime import datetime
from mock import patch
from collections import namedtuple

import synapseclient
from synapseclient.exceptions import *
from synapseclient import File, Folder, Schema, EntityViewSchema
from synapseclient.utils import id_of
from synapseclient.table import Column, RowSet, Row, as_table_columns, Table, PartialRowset, PartialRow

import integration
from integration import schedule_for_cleanup, QUERY_TIMEOUT_SEC


def setup(module):
    module.syn = integration.syn
    module.project = integration.project

    module.syn.table_query_timeout = 423

def test_create_and_update_file_view():

    ## Create a folder
    folder = Folder(str(uuid.uuid4()), parent=project, description='creating a file-view')
    folder = syn.store(folder)

    ## Create dummy file with annotations in our folder
    path = utils.make_bogus_data_file()
    file_annotations = dict(fileFormat='jpg', dataType='image', artist='Banksy',
                            medium='print', title='Girl With Ballon')
    schedule_for_cleanup(path)
    a_file = File(path, parent=folder, annotations=file_annotations)
    a_file = syn.store(a_file)
    schedule_for_cleanup(a_file)

    # Add new columns for the annotations on this file and get their IDs
    my_added_cols = [syn.store(synapseclient.Column(name=k, columnType="STRING")) for k in file_annotations.keys()]
    my_added_cols_ids = [c['id'] for c in my_added_cols]
    view_default_ids = [c['id'] for c in syn._get_default_entity_view_columns('file')]
    col_ids = my_added_cols_ids + view_default_ids
    scopeIds = [folder['id'].lstrip('syn')]

    ## Create an empty entity-view with defined scope as folder

    entity_view = EntityViewSchema(name=str(uuid.uuid4()), scopeIds=scopeIds, addDefaultViewColumns=True, addAnnotationColumns=False, type='file', columns=my_added_cols, parent=project)

    entity_view = syn.store(entity_view)
    schedule_for_cleanup(entity_view)

    assert_equals(set(scopeIds), set(entity_view.scopeIds))
    assert_equals(set(col_ids), set(entity_view.columnIds))
    assert_equals('file', entity_view.type)

    ## get the current view-schema
    view = syn.tableQuery("select * from %s" % entity_view.id)
    schedule_for_cleanup(view.filepath)

    view_dict = list(csv.DictReader(io.open(view.filepath, encoding="utf-8", newline='')))

    # check that all of the annotations were retrieved from the view
    assert set(file_annotations.keys()).issubset(set(view_dict[0].keys()))

    updated_a_file = syn.get(a_file.id, downloadFile=False)

    # Check that the values are the same as what was set
    # Both in the view and on the entity itself
    for k, v in file_annotations.items():
        assert_equals(view_dict[0][k], v)
        assert_equals(updated_a_file.annotations[k][0], v)

    # Make a change to the view and store
    view_dict[0]['fileFormat'] = 'PNG'

    with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as temp:
        schedule_for_cleanup(temp.name)
        temp_filename = temp.name

    with io.open(temp_filename, mode='w', encoding="utf-8", newline='') as temp_file:
        dw = csv.DictWriter(temp_file, fieldnames=view_dict[0].keys(),
                            quoting=csv.QUOTE_NONNUMERIC,
                            lineterminator=str(os.linesep))
        dw.writeheader()
        dw.writerows(view_dict)
        temp_file.flush()
    new_view = syn.store(synapseclient.Table(entity_view.id, temp_filename))
    new_view_dict = list(csv.DictReader(io.open(temp_filename, encoding="utf-8", newline='')))
    assert_equals(new_view_dict[0]['fileFormat'], 'PNG')

    #query for the change
    start_time = time.time()

    new_view_results = syn.tableQuery("select * from %s" % entity_view.id)
    schedule_for_cleanup(new_view_results.filepath)
    new_view_dict = list(csv.DictReader(io.open(new_view_results.filepath, encoding="utf-8", newline='')))
    #query until change is seen.
    while new_view_dict[0]['fileFormat'] != 'PNG':
        #check timeout
        assert_less(time.time() - start_time, QUERY_TIMEOUT_SEC)
        #query again
        new_view_results = syn.tableQuery("select * from %s" % entity_view.id)
        new_view_dict = list(csv.DictReader(io.open(new_view_results.filepath, encoding="utf-8", newline='')))
    #paranoid check
    assert_equals(new_view_dict[0]['fileFormat'], 'PNG')


def test_entity_view_add_annotation_columns():
    folder1 = syn.store(Folder(name=str(uuid.uuid4()) + 'test_entity_view_add_annotation_columns_proj1', parent=project, annotations={'strAnno':'str1', 'intAnno':1, 'floatAnno':1.1}))
    folder2 = syn.store(Folder(name=str(uuid.uuid4()) + 'test_entity_view_add_annotation_columns_proj2', parent=project,annotations={'dateAnno':datetime.now(), 'strAnno':'str2', 'intAnno':2}))
    schedule_for_cleanup(folder1)
    schedule_for_cleanup(folder2)
    scopeIds = [utils.id_of(folder1), utils.id_of(folder2)]

    entity_view = EntityViewSchema(name=str(uuid.uuid4()), scopeIds=scopeIds, addDefaultViewColumns=False, addAnnotationColumns=True, type='project', parent=project)
    assert_true(entity_view['addAnnotationColumns'])

    #For some reason this call is eventually consistent but not immediately consistent. so we just wait till the size returned is correct
    expected_column_types = {'dateAnno': 'DATE', 'intAnno': 'INTEGER', 'strAnno': 'STRING', 'floatAnno': 'DOUBLE'}
    columns = syn._get_annotation_entity_view_columns(scopeIds, 'project')

    start_time = time.time()
    while len(columns) != len(expected_column_types):
        assert_less(time.time() - start_time, QUERY_TIMEOUT_SEC)

        columns = syn._get_annotation_entity_view_columns(scopeIds, 'project')
        time.sleep(2)

    entity_view = syn.store(entity_view)
    assert_false(entity_view['addAnnotationColumns'])

    view_column_types = {column['name']:column['columnType'] for column in syn.getColumns(entity_view)}
    assert_dict_equal(expected_column_types, view_column_types)

    #add another annotation to the project and make sure that EntityViewSchema only adds one moe column
    folder1['anotherAnnotation'] = 'I need healing!'
    folder1 = syn.store(folder1)

    prev_columns = list(entity_view.columnIds)
    # sometimes annotation columns are not immediately updated so we wait for it to update in a loop
    start_time = time.time()
    while len(entity_view.columnIds) != len(prev_columns) + 1:
        assert_less(time.time() - start_time, QUERY_TIMEOUT_SEC)
        entity_view.addAnnotationColumns = True
        entity_view = syn.store(entity_view)

    expected_column_types.update({'anotherAnnotation': 'STRING'})
    view_column_types = {column['name']:column['columnType'] for column in syn.getColumns(entity_view)}
    assert_dict_equal(expected_column_types, view_column_types)


def test_rowset_tables():
    cols = [Column(name='name', columnType='STRING', maximumSize=1000),
            Column(name='foo', columnType='STRING', enumValues=['foo', 'bar', 'bat']),
            Column(name='x', columnType='DOUBLE'),
            Column(name='age', columnType='INTEGER'),
            Column(name='cartoon', columnType='BOOLEAN'),
            Column(name='description', columnType='LARGETEXT')]

    schema1 = syn.store(Schema(name='Foo Table', columns=cols, parent=project))

    data1 =[['Chris',  'bar', 11.23, 45, False, 'a'],
            ['Jen',    'bat', 14.56, 40, False, 'b'],
            ['Jane',   'bat', 17.89,  6, False, 'c'*1002],
            ['Henry',  'bar', 10.12,  1, False, 'd']]
    row_reference_set1 = syn.store(
        RowSet(schema=schema1, rows=[Row(r) for r in data1]))
    assert len(row_reference_set1['rows']) == 4

def test_tables_csv():

    ## Define schema
    cols = [Column(id='1', name='Name', columnType='STRING'),
            Column(id='2', name='Born', columnType='INTEGER'),
            Column(id='3', name='Hipness', columnType='DOUBLE'),
            Column(id='4', name='Living', columnType='BOOLEAN')]

    schema = Schema(name='Jazz Guys', columns=cols, parent=project)

    data = [["<NAME>",  1926, 8.65, False],
            ["<NAME>",    1926, 9.87, False],<commit_msg>fix table test<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from backports import csv
import io
import filecmp
import math
import os
import random
import sys
import tempfile
import time
import uuid
import six
from builtins import zip
from nose.tools import assert_equals, assert_less, assert_not_equal, assert_dict_equal, assert_false, assert_true
from datetime import datetime
from mock import patch
from collections import namedtuple

import synapseclient
from synapseclient.exceptions import *
from synapseclient import File, Folder, Schema, EntityViewSchema
from synapseclient.utils import id_of
from synapseclient.table import Column, RowSet, Row, as_table_columns, Table, PartialRowset, PartialRow

import integration
from integration import schedule_for_cleanup, QUERY_TIMEOUT_SEC


def setup(module):
    module.syn = integration.syn
    module.project = integration.project

    module.syn.table_query_timeout = 423

def test_create_and_update_file_view():

    ## Create a folder
    folder = Folder(str(uuid.uuid4()), parent=project, description='creating a file-view')
    folder = syn.store(folder)

    ## Create dummy file with annotations in our folder
    path = utils.make_bogus_data_file()
    file_annotations = dict(fileFormat='jpg', dataType='image', artist='Banksy',
                            medium='print', title='Girl With Ballon')
    schedule_for_cleanup(path)
    a_file = File(path, parent=folder, annotations=file_annotations)
    a_file = syn.store(a_file)
    schedule_for_cleanup(a_file)

    # Add new columns for the annotations on this file and get their IDs
    my_added_cols = [syn.store(synapseclient.Column(name=k, columnType="STRING")) for k in file_annotations.keys()]
    my_added_cols_ids = [c['id'] for c in my_added_cols]
    view_default_ids = [c['id'] for c in syn._get_default_entity_view_columns('file')]
    col_ids = my_added_cols_ids + view_default_ids
    scopeIds = [folder['id'].lstrip('syn')]

    ## Create an empty entity-view with defined scope as folder

    entity_view = EntityViewSchema(name=str(uuid.uuid4()), scopeIds=scopeIds, addDefaultViewColumns=True, addAnnotationColumns=False, type='file', columns=my_added_cols, parent=project)

    entity_view = syn.store(entity_view)
    schedule_for_cleanup(entity_view)

    assert_equals(set(scopeIds), set(entity_view.scopeIds))
    assert_equals(set(col_ids), set(entity_view.columnIds))
    assert_equals('file', entity_view.type)

    ## get the current view-schema
    view = syn.tableQuery("select * from %s" % entity_view.id)
    schedule_for_cleanup(view.filepath)

    view_dict = list(csv.DictReader(io.open(view.filepath, encoding="utf-8", newline='')))

    # check that all of the annotations were retrieved from the view
    assert set(file_annotations.keys()).issubset(set(view_dict[0].keys()))

    updated_a_file = syn.get(a_file.id, downloadFile=False)

    # Check that the values are the same as what was set
    # Both in the view and on the entity itself
    for k, v in file_annotations.items():
        assert_equals(view_dict[0][k], v)
        assert_equals(updated_a_file.annotations[k][0], v)

    # Make a change to the view and store
    view_dict[0]['fileFormat'] = 'PNG'

    with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as temp:
        schedule_for_cleanup(temp.name)
        temp_filename = temp.name

    with io.open(temp_filename, mode='w', encoding="utf-8", newline='') as temp_file:
        dw = csv.DictWriter(temp_file, fieldnames=view_dict[0].keys(),
                            quoting=csv.QUOTE_NONNUMERIC,
                            lineterminator=str(os.linesep))
        dw.writeheader()
        dw.writerows(view_dict)
        temp_file.flush()
    new_view = syn.store(synapseclient.Table(entity_view.id, temp_filename))
    new_view_dict = list(csv.DictReader(io.open(temp_filename, encoding="utf-8", newline='')))
    assert_equals(new_view_dict[0]['fileFormat'], 'PNG')

    #query for the change
    start_time = time.time()

    new_view_results = syn.tableQuery("select * from %s" % entity_view.id)
    schedule_for_cleanup(new_view_results.filepath)
    new_view_dict = list(csv.DictReader(io.open(new_view_results.filepath, encoding="utf-8", newline='')))
    #query until change is seen.
    while new_view_dict[0]['fileFormat'] != 'PNG':
        #check timeout
        assert_less(time.time() - start_time, QUERY_TIMEOUT_SEC)
        #query again
        new_view_results = syn.tableQuery("select * from %s" % entity_view.id)
        new_view_dict = list(csv.DictReader(io.open(new_view_results.filepath, encoding="utf-8", newline='')))
    #paranoid check
    assert_equals(new_view_dict[0]['fileFormat'], 'PNG')


def test_entity_view_add_annotation_columns():
    folder1 = syn.store(Folder(name=str(uuid.uuid4()) + 'test_entity_view_add_annotation_columns_proj1', parent=project, annotations={'strAnno':'str1', 'intAnno':1, 'floatAnno':1.1}))
    folder2 = syn.store(Folder(name=str(uuid.uuid4()) + 'test_entity_view_add_annotation_columns_proj2', parent=project,annotations={'dateAnno':datetime.now(), 'strAnno':'str2', 'intAnno':2}))
    schedule_for_cleanup(folder1)
    schedule_for_cleanup(folder2)
    scopeIds = [utils.id_of(folder1), utils.id_of(folder2)]

    entity_view = EntityViewSchema(name=str(uuid.uuid4()), scopeIds=scopeIds, addDefaultViewColumns=False, addAnnotationColumns=True, type='project', parent=project)
    assert_true(entity_view['addAnnotationColumns'])

    #For some reason this call is eventually consistent but not immediately consistent. so we just wait till the size returned is correct
    expected_column_types = {'dateAnno': 'DATE', 'intAnno': 'INTEGER', 'strAnno': 'STRING', 'floatAnno': 'DOUBLE'}
    columns = syn._get_annotation_entity_view_columns(scopeIds, 'project')

    start_time = time.time()
    while len(columns) != len(expected_column_types):
        assert_less(time.time() - start_time, QUERY_TIMEOUT_SEC)

        columns = syn._get_annotation_entity_view_columns(scopeIds, 'project')
        time.sleep(2)

    entity_view = syn.store(entity_view)
    assert_false(entity_view['addAnnotationColumns'])

    view_column_types = {column['name']:column['columnType'] for column in syn.getColumns(entity_view)}
    assert_dict_equal(expected_column_types, view_column_types)

    #add another annotation to the project and make sure that EntityViewSchema only adds one moe column
    folder1['anotherAnnotation'] = 'I need healing!'
    folder1 = syn.store(folder1)

    prev_columns = list(entity_view.columnIds)
    # sometimes annotation columns are not immediately updated so we wait for it to update in a loop
    start_time = time.time()
    while len(entity_view.columnIds) != len(prev_columns) + 1:
        assert_less(time.time() - start_time, QUERY_TIMEOUT_SEC)
        entity_view.addAnnotationColumns = True
        entity_view = syn.store(entity_view)

    expected_column_types.update({'anotherAnnotation': 'STRING'})
    view_column_types = {column['name']:column['columnType'] for column in syn.getColumns(entity_view)}
    assert_dict_equal(expected_column_types, view_column_types)


def test_rowset_tables():
    cols = [Column(name='name', columnType='STRING', maximumSize=1000),
            Column(name='foo', columnType='STRING', enumValues=['foo', 'bar', 'bat']),
            Column(name='x', columnType='DOUBLE'),
            Column(name='age', columnType='INTEGER'),
            Column(name='cartoon', columnType='BOOLEAN'),
            Column(name='description', columnType='LARGETEXT')]

    schema1 = syn.store(Schema(name='Foo Table', columns=cols, parent=project))

    data1 =[['Chris',  'bar', 11.23, 45, False, 'a'],
            ['Jen',    'bat', 14.56, 40, False, 'b'],
            ['Jane',   'bat', 17.89,  6, False, 'c'*1002],
            ['Henry',  'bar', 10.12,  1, False, 'd']]
    row_reference_set1 = syn.store(
        RowSet(schema=schema1, rows=[Row(r) for r in data1]))
    assert len(row_reference_set1['rows']) == 4

def test_tables_csv():

    ## Define schema
    cols = [Column(name='Name', columnType='STRING'),
            Column(name='Born', columnType='INTEGER'),
            Column(name='Hipness', columnType='DOUBLE'),
            Column(name='Living', columnType='BOOLEAN')]

    schema = Schema(name='Jazz Guys', columns=cols, parent=project)

    data = [["<NAME>",  1926, 8.65, False],
            ["<NAME>",    1926, 9.87, False],
{'fraction_non_alphanumeric': 0.12186755043897901, 'fraction_numerical': 0.010930868073725217, 'mean_word_length': 4.417322834645669, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5087195', 'n_tokens_mistral': 5457, 'n_tokens_neox': 5276, 'n_words': 1189}