<commit_before>#############################################################################
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
#############################################################################
#
#  Project Name        :    IEEE 802.11 Timeline Tool#                                                                            *
#
#  Author              :    <NAME>
#
#############################################################################

from util.io import from_isodatetime, flatten
from util.tasks import run_test_task_queue
from util.models import SiteURLs
from util.db import bulk_delete
from ballot.models import Ballot, DenormalizedBallot, BallotBacklog
from project.models import Project, DenormalizedProject, ProjectBacklog
from timeline.models import DenormalizedProjectBallots, ProjectBallotsBacklog
from report.models import MeetingReport

from django.test import TestCase
from django.core.urlresolvers import reverse
from django.db.models.fields import URLField

import datetime, decimal
import unittest

class LoginBasedTest(TestCase):    
    def _check_page(self,url):
        response = self.client.get(url)
        # not logged in, should redirect to login page
        self.failUnlessEqual(response.status_code, 302)
        login = self.client.login(username='test', password='password')
        self.failUnless(login, 'Could not log in')
        response = self.client.get(url)
        self.failUnlessEqual(response.status_code, 200)
        return response
        
class UtilTest(unittest.TestCase):
    def test_from_isodatetime(self):
        """
        Tests converting from ISO 8601 string to Python datetime / timedelta objects
        """
        tests = [
                 ('2009-02-27T10:00:00Z', datetime.datetime(2009,2,27,10,0,0) ),
                 ('PT14H00M00S', datetime.timedelta(hours=14) ),
                 ('PT00H45M00S', datetime.timedelta(minutes=45) ),
                 ('PT01:45:19', datetime.timedelta(hours=1,minutes=45,seconds=19) ),
                 ]
        for test in tests:
            tc = from_isodatetime(test[0])
            self.failUnlessEqual(tc,test[1])

    def test_flatten(self):
        """
        Tests converting Python objects to form suitable for storage
        """
        tests = [
                 ([1,2,3],[1,2,3]),
                 ([1,2L,3],[1,'2',3]),
                 ( [1, 2L, 3, datetime.datetime(2009, 8, 13, 10, 52, 8, 734209)], [1, '2', 3, '2009-08-13T10:52:08.734209Z']),
                 ( (1, 2L, 3, datetime.datetime(2009, 8, 13, 10, 52, 8, 734209)), (1, '2', 3, '2009-08-13T10:52:08.734209Z') ),
                 ( [decimal.Decimal('5.2'), u'Hello World', datetime.date(2009, 8, 13)], ['5.2', 'Hello World', '2009-08-13'] ),
                 ({'a': decimal.Decimal('5.2'), 'c': datetime.date(2009, 8, 13), 'b': u'Hello World'}, {'a': '5.2', 'c': '2009-08-13', 'b': 'Hello World'}),
                 ]
        for test in tests:
            f = flatten(test[0])
            self.failUnlessEqual(f,test[1])
            
class ImportPageTest(TestCase):
    NOT_IDEMPOTENT = []

    def setUp(self):
        from util import tasks 
        tasks._test_task_queue = []
        tasks._completed_tasks = []

    def _post_import(self,url,filename,length=0, wipe_projects=False, wipe_ballots=False):
        testfile = open(filename,'r')
        data = {
                'wipe_projects' : wipe_projects,
                'wipe_ballots': wipe_ballots,
                'import' : 'Import',
                'file' : testfile
                }
        try:
            response = self.client.post(url, data, follow=True)
        except:
            raise
        finally:
            testfile.close()
        progress = response.context['progress']
        progress_url = reverse('util.views.import_progress',args=[progress.pk])
        self.failUnlessEqual(response.redirect_chain[0][1],302)
        self.failUnless(response.redirect_chain[0][0].endswith(progress_url))
        #print response.content
        #self.failIf(response.context['errors'])
        #print response.content
        if length:
            self.assertContains(response, 'input file contains %d line'%length)
        #self.assertContains(response, 'Finished Importing')
        while progress.finished is None:
            run_test_task_queue(self.client)
            response = self.client.get(progress_url)
            self.failUnlessEqual(response.status_code, 200)
            progress = response.context['progress']
        done_url = reverse('util.views.import_done',args=[progress.pk])
        response = self.client.get(done_url)
        self.failUnlessEqual(response.status_code, 200)
        return response
        
class ImportHtmlPageTest(ImportPageTest):
    fixtures = ['site.json','projects.json']
    LB_BALLOT_HTML = 'util/fixtures/LetterBallots.htm'
    SB_BALLOT_HTML = 'util/fixtures/SponsorBallots.htm'
    
    def test_html_import(self):
        """
        Tests import page
        """

        url = reverse('util.views.import_page',args=[])
        response = self.client.get(url)
        # not logged in, should redirect to login page
        self.failUnlessEqual(response.status_code, 302)

        login = self.client.login(username='test', password='password')
        self.failUnless(login, 'Could not log in')
        response = self.client.get(url)
        self.failUnlessEqual(response.status_code, 200)
        self.assertContains(response, '<input type="submit" name="submit"')
        
        response = self._post_import(url, self.LB_BALLOT_HTML)

        #print 'results',response.context['results']
        counts = {}
        tmodels = { Ballot:0, Project: 0 }
        for m in tmodels.keys():
            self.failIfEqual(m.objects.count(),tmodels[m],msg='%s model has %d items, should have more than %d items'%
                                 (m._meta.verbose_name,m.objects.count(),tmodels[m]))
            counts[m] = m.objects.count()
        tmodels[Ballot] = counts[Ballot]
        
        # Check that a second attempt to import the data does not cause errors and does not add any new objects to the database
        response = self._post_import(url, self.LB_BALLOT_HTML)
        for m in tmodels.keys():
            self.failUnlessEqual(m.objects.count(),counts[m],msg='%s model had %d items, but after a second import has %d'%
                                 (m._meta.verbose_name,counts[m],m.objects.count()))
        
        response = self._post_import(url, self.SB_BALLOT_HTML)
        for m in tmodels.keys():
            self.failIfEqual(m.objects.count(),tmodels[m],msg='%s model has %d items, should have more than %d items'%
                                 (m._meta.verbose_name,m.objects.count(),tmodels[m]))
            counts[m] = m.objects.count()
            
        # Check that a second attempt to import the data does not cause errors 
        # and does not add any new objects to the database
        response = self._post_import(url, self.SB_BALLOT_HTML)
        for m in tmodels.keys():
            self.failUnlessEqual(m.objects.count(),counts[m],msg='%s model had %d items, but after a second import has %d'%
                                 (m._meta.verbose_name,counts[m],m.objects.count()))
     
class ImportCsvPageTest(ImportPageTest):
    fixtures = ['site.json']
    TESTFILE = ('util/fixtures/timeline-2010-11-02-1439.csv' ,187)
    MODELS = [Ballot, Project]<commit_msg>Added test for all the date formats that are supported. Added test for importing HTML projects from a timeline.htm file. Change LoginBasedTest to only check for login once per test.<commit_after>#############################################################################
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
#############################################################################
#
#  Project Name        :    IEEE 802.11 Timeline Tool#                                                                            *
#
#  Author              :    <NAME>
#
#############################################################################

from util.io import from_isodatetime, flatten, parse_date
from util.tasks import run_test_task_queue
from util.models import SiteURLs
from util.db import bulk_delete
from util.views import import_page, import_progress
from ballot.models import Ballot, DenormalizedBallot, BallotBacklog
from project.models import Project, DenormalizedProject, ProjectBacklog
from timeline.models import DenormalizedProjectBallots, ProjectBallotsBacklog
from report.models import MeetingReport

from django.test import TestCase
from django.test.client import RequestFactory
from django.core.urlresolvers import reverse
from django.contrib.auth import authenticate
from django.db.models.fields import URLField
from django.conf import settings
from django.utils import http

import datetime, decimal
import unittest

class LoginBasedTest(TestCase):    
    def setUp(self):
        self.login = None
        
    def _check_page(self,url, status_code=200):
        response = self.client.get(url)
        if not self.login:
            # not logged in, should redirect to login page
            self.failUnlessEqual(response.status_code, 302)
            self.assertRedirects(response,settings.LOGIN_URL+'?next='+http.urlquote(url))
            self.login = self.client.login(username='test', password='password')
            self.failUnless(self.login, 'Could not log in')
        response = self.client.get(url)
        self.failUnlessEqual(response.status_code, status_code)
        return response
        
class UtilTest(unittest.TestCase):
    def test_from_isodatetime(self):
        """
        Tests converting from ISO 8601 string to Python datetime / timedelta objects
        """
        tests = [
                 ('2009-02-27T10:00:00Z', datetime.datetime(2009,2,27,10,0,0) ),
                 ('PT14H00M00S', datetime.timedelta(hours=14) ),
                 ('PT00H45M00S', datetime.timedelta(minutes=45) ),
                 ('PT01:45:19', datetime.timedelta(hours=1,minutes=45,seconds=19) ),
                 ]
        for test in tests:
            tc = from_isodatetime(test[0])
            self.failUnlessEqual(tc,test[1])

    def test_flatten(self):
        """
        Tests converting Python objects to form suitable for storage
        """
        tests = [
                 ([1,2,3],[1,2,3]),
                 ([1,2L,3],[1,'2',3]),
                 ( [1, 2L, 3, datetime.datetime(2009, 8, 13, 10, 52, 8, 734209)], [1, '2', 3, '2009-08-13T10:52:08.734209Z']),
                 ( (1, 2L, 3, datetime.datetime(2009, 8, 13, 10, 52, 8, 734209)), (1, '2', 3, '2009-08-13T10:52:08.734209Z') ),
                 ( [decimal.Decimal('5.2'), u'Hello World', datetime.date(2009, 8, 13)], ['5.2', 'Hello World', '2009-08-13'] ),
                 ({'a': decimal.Decimal('5.2'), 'c': datetime.date(2009, 8, 13), 'b': u'Hello World'}, {'a': '5.2', 'c': '2009-08-13', 'b': 'Hello World'}),
                 ]
        for test in tests:
            f = flatten(test[0])
            self.failUnlessEqual(f,test[1])
            
    def test_date_parse(self):
        dates = [
                 ('2008-05-03', datetime.datetime(2008,5,3)), 
                 ('05/30/06', datetime.datetime(2006,5,30)), 
                 ('05/30/2006', datetime.datetime(2006,5,30)), 
                 ('Mon Sep 27, 2010', datetime.datetime(2010,9,27)),
                 ('Sep 16 2007 - 23:59 ET', datetime.datetime(2007,9,17,4,59)),
                 ('October 16 2007 - 23:59 ET', datetime.datetime(2007,10,17,4,59)),
                 ('May 2, 2007 - 23:59 ET', datetime.datetime(2007,5,3,4,59)),
                 ('Sep-14', datetime.datetime(2014,9,1)),
                 ('09/xx/14', datetime.datetime(2014,9,1)), 
                 ('May 14', datetime.datetime(2014,5,1)), 
                 ('Oct 2014', datetime.datetime(2014,10,1)), 
                 ('October 7 2014', datetime.datetime(2014,10,7)) ,
                 ('Jul 26 2013', datetime.datetime(2013,7,26)) 
                 ]
        for s,d in dates:
            v = parse_date(s)
            self.failIfEqual(v,None,s)
            self.failUnlessEqual(v,d,s)
                   
            
class ImportPageTest(LoginBasedTest):
    NOT_IDEMPOTENT = []

    #def setUp(self):
    #    super(ImportPageTest,self).setUp()
    #    self.user = authenticate(username='test', password='password')
    #    self.factory = RequestFactory()

    def _post_import(self,url,filename,length=0, wipe_projects=False, wipe_ballots=False):
        testfile = open(filename,'r')
        data = {
                'wipe_projects' : wipe_projects,
                'wipe_ballots': wipe_ballots,
                'import' : 'Import',
                'file' : testfile
                }
        try:
            response = self.client.post(url, data, follow=True)
            #request = self.factory.post(url, data)
            #request.user = self.user 
            #response = import_page(request, next='/')
        except:
            raise
        finally:
            testfile.close()
        #progress_url =  response.get('Location')
        self.failUnlessEqual(response.redirect_chain[0][1],302)
        progress = response.context['progress']
        progress_url = reverse('util.views.import_progress',args=[progress.pk])
        self.failUnless(response.redirect_chain[0][0].endswith(progress_url))
        #self.failIf(response.context['errors'])
        if length:
            self.assertContains(response, 'input file contains %d line'%length)
        while progress.finished is None:
            run_test_task_queue(self.client)
            response = self.client.get(progress_url)
            #request = self.factory.get(progress_url)
            #request.user = self.user 
            #response = import_progress(request, progress.pk)
            self.failUnlessEqual(response.status_code, 200)
            progress = response.context['progress']
        done_url = reverse('util.views.import_done',args=[progress.pk])
        response = self.client.get(done_url)
        self.failUnlessEqual(response.status_code, 200)
        return response
        
class ImportHtmlBallotTest(ImportPageTest):
    fixtures = ['site.json','projects.json']
    LB_BALLOT_HTML = 'util/fixtures/LetterBallots.htm'
    SB_BALLOT_HTML = 'util/fixtures/SponsorBallots.htm'
    
    def test_ballot_import(self):
        """
        Tests import page
        """
        url = reverse('util.views.import_page',args=[])
        response = self._check_page(url)
        self.assertContains(response, '<input type="submit" name="submit"')
        
        response = self._post_import(url, self.LB_BALLOT_HTML)
        print response
        counts = {}
        tmodels = { Ballot:0, Project: 0 }
        for m in tmodels.keys():
            self.failIfEqual(m.objects.count(),tmodels[m],msg='%s model has %d items, should have more than %d items'%
                                 (m._meta.verbose_name,m.objects.count(),tmodels[m]))
            counts[m] = m.objects.count()
        tmodels[Ballot] = counts[Ballot]
        
        # Check that a second attempt to import the data does not cause errors and does not add any new objects to the database
        response = self._post_import(url, self.LB_BALLOT_HTML)
        for m in tmodels.keys():
            self.failUnlessEqual(m.objects.count(),counts[m],msg='%s model had %d items, but after a second import has %d'%
                                 (m._meta.verbose_name,counts[m],m.objects.count()))
        
        response = self._post_import(url, self.SB_BALLOT_HTML)
        for m in tmodels.keys():
            self.failIfEqual(m.objects.count(),tmodels[m],msg='%s model has %d items, should have more than %d items'%
                                 (m._meta.verbose_name,m.objects.count(),tmodels[m]))
            counts[m] = m.objects.count()
            
        # Check that a second attempt to import the data does not cause errors 
        # and does not add any new objects to the database
        response = self._post_import(url, self.SB_BALLOT_HTML)
        for m in tmodels.keys():
            self.failUnlessEqual(m.objects.count(),counts[m],msg='%s model had %d items, but after a second import has %d'%
                                 (m._meta.verbose_name,counts[m],m.objects.count()))
            
class ImportHtmlTimelineTest(ImportPageTest):
    fixtures = ['site.json']
    LB_BALLOT_HTML = 'util/fixtures/LetterBallots.htm'
    SB_BALLOT_HTML = 'util/fixtures/SponsorBallots.htm'
    TIMELINE_HTML = 'util/fixtures/Timelines.htm'
    
    def test_timeline_import(self):
        """
        Tests import page with a timelines.html
        """
        url = reverse('util.views.import_page',args=[])
        self._check_page(url)
        response = self._post_import(url, self.TIMELINE_HTML)
        #print 'response',response
        self.assertEqual(Project.objects.count(),37)

class ImportCsvPageTest(ImportPageTest):
    fixtures = ['site.json']
    TESTFILE = ('util/fixtures/timeline-2010-11-02-1439.csv' ,187)
    MODELS = [Ballot, Project]
{'fraction_non_alphanumeric': 0.14180377483079293, 'fraction_numerical': 0.0369779342981346, 'mean_word_length': 2.4749521988527725, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5165465', 'n_tokens_mistral': 5723, 'n_tokens_neox': 5176, 'n_words': 1220}