<commit_before>from django.test import TestCase
from v1.processors import eventpage
import collections

class ImportDataTest(TestCase):
    def test_eventpage_convert(self):
        imported_data = collections.OrderedDict([('_id', u'consumer-advisory-board-meeting-in-itta-bena-miss'), ('agenda', [{u'speakers': [{u'url': u'http://www.consumerfinance.gov/the-bureau/about-rich-cordray/', u'name': u'Director, <NAME>'}, {u'name': u'CAB Staff Director, <NAME>'}, {u'name': u'CAB Chair, <NAME>'}], u'location': u'Social Science Auditorium', u'ending_time': {u'date': u'2016-02-22T11:00:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T10:00:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Welcome and meeting overview (Closed Session)'}, {u'speakers': [{u'url': u'http://www.consumerfinance.gov/the-bureau/about-rich-cordray/', u'name': u'Director, <NAME>'}, {u'name': u'Deputy Director, <NAME>'}, {u'name': u'Assistant Director, Office of Financial Empowerment, <NAME>'}, {u'name': u'Mortgage Data Assets Team Lead, <NAME>'}], u'location': u'Social Science Auditorium ', u'ending_time': {u'date': u'2016-02-22T13:00:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T11:00:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'*CAB Public Session: Access to Credit, Information and Financial Resources (Open Session)'}, {u'speakers': [{u'name': u'CAB Chair, <NAME>\xf1onez'}], u'location': u'Social Science Auditorium ', u'ending_time': {u'date': u'2016-02-22T14:30:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T13:00:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Working lunch: CAB meeting resumes closed session'}, {u'speakers': [{u'name': u'CAB Chair, <NAME>\xf1onez'}], u'location': u'Social Science Auditorium ', u'ending_time': {u'date': u'2016-02-22T15:30:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T14:30:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Year in Review (Closed Session)'}, {u'speakers': [{u'name': u'CAB Member, <NAME>'}, {u'name': u'CAB Member, <NAME>'}], u'location': u'Social Science Auditorium', u'ending_time': {u'date': u'2016-02-22T17:30:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T15:30:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Policy Discussion: Trends and Themes in the field (Closed Session)'}, {u'location': u'Hilton Garden Inn\r\n235 West Capitol Street\r\nJackson, MS 39201', u'ending_time': {u'date': u'2016-02-22T22:30:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T08:30:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'CAB Committee meetings (Closed Session)\r\n'}, {u'speakers': [{u'name': u'Mortgage Data Assets Team Lead, <NAME>'}], u'location': u'Hilton Garden Inn', u'ending_time': {u'date': u'2016-02-22T11:30:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T10:30:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Home Mortgage Disclosure Act discussion (Closed Session)'}, {u'speakers': [{u'name': u'Financial Empowerment Program Specialist, <NAME>'}], u'location': u'Hilton Garden Inn', u'ending_time': {u'date': u'2016-02-22T14:15:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T13:30:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Your Money, Your Goals discussion (Closed Session)'}, {u'speakers': [{u'name': u'Committee Chair <NAME>'}, {u'name': u'Committee Chair <NAME>'}, {u'name': u'Committee Chair <NAME>'}, {u'name': u'Committee Chair <NAME>'}], u'location': u'Hilton Garden Inn', u'ending_time': {u'date': u'2016-02-22T15:45:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T14:15:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Committee Reports (Closed Session)'}, {u'speakers': [{u'url': u'http://www.consumerfinance.gov/the-bureau/about-rich-cordray/', u'name': u'Director, <NAME>'}, {u'name': u'Associate Director, Research, Markets and Regulations Division, <NAME>'}], u'location': u'Hilton Garden Inn', u'ending_time': {u'date': u'2016-02-22T17:15:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T16:00:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Policy Discussion (Closed Session)'}]), ('archive', {u'summary': u'<p>The\xa0CAB met with Director <NAME> to announce two initiatives: the first\xa0aimed at improving understanding of the mortgage market; and the second aimed at boosting financial empowerment for certain low- and moderate-income consumers.</p>'}), (u'attachments', []), (u'author', {u'first_name': u'Sarah', u'last_name': u'Simpson', u'name': u'<NAME>', u'url': u'', u'slug': u'sarah-simpson', u'nickname': u'sarah.simpson', u'id': 353, u'description': u''}), ('beginning_time', {'date': u'2013-09-18T10:00:00-05:00', 'timezone': u'America/Chicago'}), (u'categories', []), (u'comment_count', 0), (u'comment_status', u'closed'), (u'content', u'<p>The\xa0CAB met with Director <NAME> to announce two initiatives: the first\xa0aimed at improving understanding of the mortgage market; and the second aimed at boosting financial empowerment for certain low- and moderate-income consumers.</p>\n'), (u'date', u'2015-12-10T14:20:29-0500'), (u'dek', u''), ('ending_time', {'date': u'2013-09-19T17:30:00-05:00', 'timezone': u'America/Chicago'}), (u'excerpt', u'<p>The\xa0CAB met with Director <NAME> to announce two initiatives: the first\xa0aimed at improving understanding of the mortgage market; and the second aimed at boosting financial empowerment for certain low- and moderate-income consumers.</p>\n'), ('ics', {'dtstamp': u'2015-12-10T14:20:29-0500', 'uid': u'/event/consumer-advisory-board-meeting-in-itta-bena-miss/', 'summary': u'Consumer Advisory Board meeting in Itta Bena, Miss.', 'dtend': u'2013-09-19T17:30:00-05:00', 'dtstart': u'2013-09-18T10:00:00-05:00', 'starting_tzinfo': u'America/Chicago', 'ending_tzinfo': u'America/Chicago'}), (u'id', 44119), (u'modified', u'2016-02-22T14:48:59+0000'), ('open_graph', {}), (u'order', 0), (u'parent', 0), (u'relative_url', u'/event/consumer-advisory-board-meeting-in-itta-bena-miss/'), ('rsvp', {u'contact': [u'']}), (u'slug', u'consumer-advisory-board-meeting-in-itta-bena-miss'), (u'status', u'publish'), (u'tags', [u'Consumer Advisory Board', u'Financial Empowerment', u'HMDA', u'Mortgages']), (u'title', u'Consumer Advisory Board meeting in Itta Bena, Miss.'), (u'title_plain', u'Consumer Advisory Board meeting in Itta Bena, Miss.'), (u'type', u'event'), (u'url', u'http://www.consumerfinance.gov/event/consumer-advisory-board-meeting-in-itta-bena-miss/'), ('venue', {u'name': u'Mississippi Valley State University', u'address': {u'city': u'Itta Bena', u'state': u'MS', u'street': u'14000 Highway 82 West', u'zip_code': u' 38941'}})])
        output = eventpage.DataConverter().convert(imported_data)
        expected_output = {'agenda_items-4-value-speakers-0-value-name': u'CAB Member, <NAME>', 'agenda_items-7-value-speakers-0-value-name': u'Financial Empowerment Program Specialist, <NAME>', 'agenda_items-0-value-speakers-2-value-url': u'', 'agenda_items-1-value-speakers-3-value-name': u'Mortgage Data Assets Team Lead, <NAME>', 'agenda_items-0-value-speakers-1-deleted': u'', 'agenda_items-4-value-speakers-0-value-url': u'', 'agenda_items-7-value-start_dt': '2016-02-22 13:30', 'agenda_items-9-value-end_dt': '2016-02-22 17:15', 'sidefoot-0-value-view_more-text': 'View More', 'youtube_url': u'', 'agenda_items-0-value-speakers-2-deleted': u'', 'agenda_items-9-value-description': u'Policy Discussion (Closed Session)', 'agenda_items-3-value-speakers-count': 1, 'agenda_items-1-value-speakers-0-deleted': u'', 'agenda_items-6-value-speakers-0-deleted': u'', 'categories-INITIAL_FORMS': '0', 'agenda_items-8-value-speakers-3-value-url': u'', 'agenda_items-8-value-speakers-1-value-url': u'', 'agenda_items-2-deleted': u'', 'agenda_items-9-value-speakers-0-value-name': u'Director, <NAME>', 'agenda_items-4-value-speakers-0-order': 0, 'agenda_items-8-value-speakers-2-deleted': u'', 'agenda_items-9-type': 'item', 'agenda_items-7-order': 7, 'agenda_items-3-type': 'item', 'agenda_items-9-value-speakers-0-order': 0, 'agenda_items-0-value-speakers-0-order': 0, 'agenda_items-9-value-speakers-0-value-url': u'http://www.consumerfinance.gov/the-bureau/about-rich-cordray/', 'agenda_items-3-value-end_dt': '2016-02-22 15:30', 'agenda_items-4-value-speakers-1-value-url': u'', 'agenda_items-0-value-speakers-0-value-url': u'http://www.consumerfinance.gov/the-bureau/about-rich-cordray/', 'agenda_items-0-value-description': u'Welcome and meeting overview (Closed Session)', 'agenda_items-1-order': 1, 'date_published': '2013-09-18', 'agenda_items-6-value-speakers-0-value-url': u'', 'agenda_items-7-value-speakers-count': 1, 'sidefoot-0-value-view_more-url': '/', 'agenda_items-1-value-speakers-1-value-name': u'Deputy Director, <NAME>', 'agenda_items-6-type': 'item', 'venue_zip': u'', 'agenda_items-2-value-description': u'Working lunch: CAB meeting resumes closed session', 'agenda_items-0-value-speakers-count': 3, 'agenda_items-8-value-speakers-1-value-name': u'Committee Chair <NAME>', 'venue_suite': u'', 'categories-MAX_NUM_FORMS': '2', 'agenda_items-6-value-description': u'Home Mortgage Disclosure Act discussion (Closed Session)', 'agenda_items-7-value-speakers-0-deleted': u'', 'agenda_items-8-value-speakers-2-value-url': u'', 'agenda_items-8-value-speakers-2-order': 2, 'agenda_items-7-value-description': u'Your Money, Your Goals discussion (Closed Session)', 'agenda_items-9-value-speakers-count': 2, 'start_dt': '2013-09-18 10:00', 'tags': u'"Consumer Advisory Board", "Financial Empowerment", HMDA, Mortgages', 'agenda_items-9-value-speakers-0-deleted': u'', 'agenda_items-8-type': 'item', 'agenda_items-7-value-speakers-0-value-url': u'', 'agenda_items-1-value-speakers-3-value-url': u'', 'agenda_items-2-value-speakers-0-value-url': u'', 'agenda_items-0-value-end_dt': '2016-02-22 11:00', 'body': u'<p>The\xa0CAB met with Director <NAME> to announce two initiatives: the first\xa0aimed at improving understanding of the mortgage market; and the second aimed at boosting financial empowerment for certain low- and moderate-income consumers.</p>\n', 'agenda_items-6-value-start_dt': '2016-02-22 10:30', 'agenda_items-7-value-location': u'Hilton Garden Inn', 'agenda_items-1-type': 'item', 'categories-MIN_NUM_FORMS': '0', 'agenda_items-3-value-speakers-0-value-name': u'CAB Chair, <NAME>\xf1onez', 'agenda_items-1-value-description': u'*CAB Public Session: Access to Credit, Information and Financial Resources (Open Session)', 'agenda_items-8-value-description': u'Committee Reports (Closed Session)', 'agenda_items-8-deleted': u'', 'agenda_items-0-value-speakers-1-value-name': u'CAB Staff Director, <NAME>', 'agenda_items-2-value-speakers-0-deleted': u'', 'sidefoot-0-value-relate_events': 'on', 'agenda_items-1-value-speakers-2-value-url': u'', 'agenda_items-9-order': 9, 'venue_state': u'MS', 'sidefoot-0-order': '0', 'agenda_items-3-value-description': u'Year in Review (Closed Session)', 'venue_name': u'Mississippi Valley State University', 'agenda_items-5-value-speakers-count': 0, 'agenda_items-4-type': 'item', 'agenda_items-9-deleted': u'', 'agenda_items-3-value-start_dt': '2016-02-22 14:30', 'agenda_items-8-value-end_dt': '2016-02-22 15:45', 'agenda_items-2-value-speakers-count': 1, 'agenda_items-2-value-speakers-0-order': 0, 'agenda_items-3-value-location': u'Social Science Auditorium ', 'agenda_items-8-value-speakers-2-value-name': u'Committee Chair <NAME>', 'agenda_items-2-type': 'item', 'agenda_items-7-value-speakers-0-order': 0, 'sidefoot-count': '1', 'agenda_items-5-value-start_dt': '2016-02-22 08:30', 'agenda_items-0-value-speakers-0-deleted': u'', 'agenda_items-6-value-end_dt': '2016-02-22 11:30', 'sidefoot-0-value-relate_posts': 'on', 'agenda_items-8-value-speakers-0-order': 0, 'agenda_items-1-deleted': u'', 'agenda_items-4-value-speakers-1-value-name': u'CAB Member, <NAME>', 'agenda_items-2-value-end_dt': '2016-02-22 14:30', 'flickr_url': u'', 'agenda_items-count': 10, 'agenda_items-4-value-speakers-count': 2, 'agenda_items-4-value-start_dt': '2016-02-22 15:30', 'agenda_items-9-value-speakers-1-order': 1, 'agenda_items-8-value-speakers-count': 4, 'agenda_items-3-deleted': u'', 'agenda_items-9-value-speakers-1-value-url': u'', 'agenda_items-5-value-description': u'CAB Committee meetings (Closed Session)\r\n', 'agenda_items-9-value-start_dt': '2016-02-22 16:00', 'agenda_items-1-value-speakers-0-order': 0, 'agenda_items-4-value-speakers-1-deleted': u'', 'categories-TOTAL_FORMS': '0', 'authors': u'"<NAME>"', 'agenda_items-5-order': 5, 'sidefoot-0-deleted': '', 'agenda_items-1-value-speakers-2-value-name': u'Assistant Director, Office of Financial Empowerment, Cliff Rosenthal', 'agenda_items-7-value-end_dt': '2016-02-22 14:15', 'agenda_items-5-value-location': u'Hilton Garden Inn\r\n235 West Capitol Street\r\nJackson, MS 39201', 'agenda_items-8-value-speakers-3-value-name': u'Committee Chair <NAME>', 'agenda_items-0-value-speakers-1-value-url': u'', 'agenda_items-1-value-speakers-1-deleted': u'', 'agenda_items-0-value-start_dt': '2016-02-22 10:00', 'agenda_items-4-deleted': u'', 'agenda_items-0-value-speakers-2-value-name': u'CAB Chair, <NAME>\xf1onez', 'agenda_items-3-value-speakers-0-deleted': u'', 'agenda_items-0-value-location': u'Social Science Auditorium', 'agenda_items-7-deleted': u'', 'agenda_items-8-value-location': u'Hilton Garden Inn', 'agenda_items-1-value-speakers-1-order': 1, 'agenda_items-8-value-speakers-1-order': 1, 'agenda_items-1-value-speakers-3-order': 3, 'agenda_items-6-value-speakers-0-order': 0, 'agenda_items-1-value-speakers-3-deleted': u'', 'agenda_items-5-deleted': u'', 'agenda_items-2-value-start_dt': '2016-02-22 13:00', 'agenda_items-2-value-location': u'Social Science Auditorium ', 'agenda_items-8-value-speakers-3-deleted': u'', 'agenda_items-4-value-location': u'Social Science Auditorium', 'agenda_items-6-deleted': u'', 'agenda_items-0-type': 'item', 'agenda_items-1-value-speakers-count': 4, 'agenda_items-0-deleted': u'', 'agenda_items-8-value-speakers-0-deleted': u'', 'venue_city': u'Itta Bena', 'agenda_items-1-value-speakers-0-value-name': u'Director, <NAME>', 'agenda_items-1-value-start_dt': '2016-02-22 11:00', 'agenda_items-1-value-end_dt': '2016-02-22 13:00', 'venue_street': u'14000 Highway 82 West', 'sidefoot-0-value-limit': '3', 'agenda_items-1-value-speakers-2-order': 2, 'title': u'Consumer Advisory Board meeting in Itta Bena, Miss.', 'sidefoot-0-type': 'related_posts', 'agenda_items-4-value-speakers-0-deleted': u'', 'agenda_items-0-value-speakers-2-order': 2, 'sidefoot-0-value-relate_newsroom': 'on', 'agenda_items-0-order': 0, 'agenda_items-4-value-end_dt': '2016-02-22 17:30', 'agenda_items-2-order': 2, 'agenda_items-9-value-speakers-1-deleted': u'', 'agenda_items-1-value-location': u'Social Science Auditorium ', 'agenda_items-0-value-speakers-1-order': 1, 'archive_body': u'<p>The\xa0CAB met with Director <NAME> to announce two initiatives: the first\xa0aimed at improving understanding of the mortgage market; and the second aimed at boosting financial empowerment for certain low- and moderate-income consumers.</p>', 'agenda_items-3-order': 3, 'agenda_items-6-order': 6, 'agenda_items-6-value-location': u'Hilton Garden Inn', 'agenda_items-8-value-speakers-3-order': 3, 'agenda_items-2-value-speakers-0-value-name': u'CAB Chair, <NAME>\xf1onez', 'agenda_items-4-value-description': u'Policy Discussion: Trends and Themes in the field (Closed Session)', 'agenda_items-9-value-location': u'Hilton Garden Inn', 'agenda_items-6-value-speakers-count': 1, 'agenda_items-4-order': 4, 'slug': u'consumer-advisory-board-meeting-in-itta-bena-miss', 'agenda_items-9-value-speakers-1-value-name': u'Associate Director, Research, Markets and Regulations Division, <NAME>', 'agenda_items-8-value-speakers-0-value-url': u'', 'agenda_items-4-value-speakers-1-order': 1, 'agenda_items-1-value-speakers-0-value-url': u'http://www.consumerfinance.gov/the-bureau/about-rich-cordray/', 'agenda_items-1-value-speakers-1-value-url': u'', 'agenda_items-8-value-speakers-1-deleted': u'', 'agenda_items-8-value-start_dt': '2016-02-22 14:15', 'end_dt': '2013-09-19 17:30', 'agenda_items-5-type': 'item', 'agenda_items-0-value-speakers-0-value-name': u'Director, <NAME>', 'agenda_items-1-value-speakers-2-deleted': u'', 'agenda_items-3-value-speakers-0-order': 0, 'agenda_items-8-order': 8, 'agenda_items-8-value-speakers-0-value-name': u'Committee Chair <NAME>', 'agenda_items-5-value-end_dt': '2016-02-22 22:30', 'agenda_items-3-value-speakers-0-value-url': u'', 'agenda_items-6-value-speakers-0-value-name': u'Mortgage Data Assets Team Lead, <NAME>', 'agenda_items-7-type': 'item'}
        self.assertEqual(output, expected_output)


if __name__ == '__main__':
    unittest.main()
<commit_msg>Add test for contact convert<commit_after>from django.test import TestCase
from v1.processors import eventpage, contact
import collections

class ImportDataTest(TestCase):
    def test_contact_convert(self):
        imported_data = {u'attachments': [], u'relative_url': u'/contact/your-money-your-goals-toolkit/', u'excerpt': u'<p>Your Money, Your Goals is a toolkit for social service organizations and others to use when working with low- and moderate-income and underserved consumers. The Office of Financial Empowerment can help you get started using YMYG in your work.</p>\n', u'id': 37397, 'web': {'url': u'http://www.consumerfinance.gov/your-money-your-goals/'}, u'author': {u'first_name': u'James', u'last_name': u'Hupp', u'name': u'<NAME>', u'url': u'', u'slug': u'jhupp', u'nickname': u'jhupp', u'id': 4, u'description': u''}, u'content': u'<p>Your Money, Your Goals is a toolkit for social service organizations and others to use when working with low- and moderate-income and underserved consumers. The Office of Financial Empowerment can help you get started using YMYG in your work.</p>\n', u'comment_count': 0, u'categories': [], u'type': u'contact', 'email': [{'addr': u'<EMAIL>'}], u'status': u'publish', u'parent': 0, u'tags': [], 'phone': [], u'date': u'2015-04-19T23:47:14-0500', u'slug': u'your-money-your-goals-toolkit', u'comment_status': u'closed', u'title_plain': u'Your Money, Your Goals Toolkit', u'url': u'http://www.consumerfinance.gov/contact/your-money-your-goals-toolkit/', u'title': u'Your Money, Your Goals Toolkit', u'modified': u'2015-04-27T10:55:34+0000', u'dek': u'', '_id': u'your-money-your-goals-toolkit', u'order': 0}
        output = contact.DataConverter().convert(imported_data)
        expected_output = {'body': u'<p>Your Money, Your Goals is a toolkit for social service organizations and others to use when working with low- and moderate-income and underserved consumers. The Office of Financial Empowerment can help you get started using YMYG in your work.</p>\n', 'contact_info-0-value-emails-0-order': u'0', 'contact_info-0-value-emails-count': '1', 'contact_info-0-type': u'email', 'contact_info-0-deleted': u'', 'contact_info-0-order': '0', 'contact_info-0-value-emails-0-value-url': u'<EMAIL>', 'contact_info-0-value-emails-0-deleted': u'', 'contact_info-count': '1', 'heading': u'Your Money, Your Goals Toolkit;;your-money-your-goals-toolkit', 'contact_info-0-value-emails-0-value-text': u'email'}
        self.assertEqual(output, expected_output)


    def test_eventpage_convert(self):
        imported_data = collections.OrderedDict([('_id', u'consumer-advisory-board-meeting-in-itta-bena-miss'), ('agenda', [{u'speakers': [{u'url': u'http://www.consumerfinance.gov/the-bureau/about-rich-cordray/', u'name': u'Director, <NAME>'}, {u'name': u'CAB Staff Director, <NAME>'}, {u'name': u'CAB Chair, <NAME>\xf1onez'}], u'location': u'Social Science Auditorium', u'ending_time': {u'date': u'2016-02-22T11:00:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T10:00:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Welcome and meeting overview (Closed Session)'}, {u'speakers': [{u'url': u'http://www.consumerfinance.gov/the-bureau/about-rich-cordray/', u'name': u'Director, <NAME>'}, {u'name': u'Deputy Director, <NAME>'}, {u'name': u'Assistant Director, Office of Financial Empowerment, <NAME>'}, {u'name': u'Mortgage Data Assets Team Lead, <NAME>'}], u'location': u'Social Science Auditorium ', u'ending_time': {u'date': u'2016-02-22T13:00:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T11:00:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'*CAB Public Session: Access to Credit, Information and Financial Resources (Open Session)'}, {u'speakers': [{u'name': u'CAB Chair, <NAME>\xf1onez'}], u'location': u'Social Science Auditorium ', u'ending_time': {u'date': u'2016-02-22T14:30:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T13:00:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Working lunch: CAB meeting resumes closed session'}, {u'speakers': [{u'name': u'CAB Chair, <NAME>\xf1onez'}], u'location': u'Social Science Auditorium ', u'ending_time': {u'date': u'2016-02-22T15:30:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T14:30:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Year in Review (Closed Session)'}, {u'speakers': [{u'name': u'CAB Member, <NAME>'}, {u'name': u'CAB Member, <NAME>'}], u'location': u'Social Science Auditorium', u'ending_time': {u'date': u'2016-02-22T17:30:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T15:30:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Policy Discussion: Trends and Themes in the field (Closed Session)'}, {u'location': u'Hilton Garden Inn\r\n235 West Capitol Street\r\nJackson, MS 39201', u'ending_time': {u'date': u'2016-02-22T22:30:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T08:30:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'CAB Committee meetings (Closed Session)\r\n'}, {u'speakers': [{u'name': u'Mortgage Data Assets Team Lead, <NAME>'}], u'location': u'Hilton Garden Inn', u'ending_time': {u'date': u'2016-02-22T11:30:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T10:30:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Home Mortgage Disclosure Act discussion (Closed Session)'}, {u'speakers': [{u'name': u'Financial Empowerment Program Specialist, <NAME>'}], u'location': u'Hilton Garden Inn', u'ending_time': {u'date': u'2016-02-22T14:15:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T13:30:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Your Money, Your Goals discussion (Closed Session)'}, {u'speakers': [{u'name': u'Committee Chair <NAME>'}, {u'name': u'Committee Chair <NAME>'}, {u'name': u'Committee Chair <NAME>'}, {u'name': u'Committee Chair <NAME>'}], u'location': u'Hilton Garden Inn', u'ending_time': {u'date': u'2016-02-22T15:45:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T14:15:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Committee Reports (Closed Session)'}, {u'speakers': [{u'url': u'http://www.consumerfinance.gov/the-bureau/about-rich-cordray/', u'name': u'Director, <NAME>'}, {u'name': u'Associate Director, Research, Markets and Regulations Division, <NAME>'}], u'location': u'Hilton Garden Inn', u'ending_time': {u'date': u'2016-02-22T17:15:00-06:00', u'timezone': u'America/Chicago'}, u'beginning_time': {u'date': u'2016-02-22T16:00:00-06:00', u'timezone': u'America/Chicago'}, u'desc': u'Policy Discussion (Closed Session)'}]), ('archive', {u'summary': u'<p>The\xa0CAB met with Director <NAME> to announce two initiatives: the first\xa0aimed at improving understanding of the mortgage market; and the second aimed at boosting financial empowerment for certain low- and moderate-income consumers.</p>'}), (u'attachments', []), (u'author', {u'first_name': u'Sarah', u'last_name': u'Simpson', u'name': u'<NAME>', u'url': u'', u'slug': u'sarah-simpson', u'nickname': u'sarah.simpson', u'id': 353, u'description': u''}), ('beginning_time', {'date': u'2013-09-18T10:00:00-05:00', 'timezone': u'America/Chicago'}), (u'categories', []), (u'comment_count', 0), (u'comment_status', u'closed'), (u'content', u'<p>The\xa0CAB met with Director <NAME> to announce two initiatives: the first\xa0aimed at improving understanding of the mortgage market; and the second aimed at boosting financial empowerment for certain low- and moderate-income consumers.</p>\n'), (u'date', u'2015-12-10T14:20:29-0500'), (u'dek', u''), ('ending_time', {'date': u'2013-09-19T17:30:00-05:00', 'timezone': u'America/Chicago'}), (u'excerpt', u'<p>The\xa0CAB met with Director Richard Cordray to announce two initiatives: the first\xa0aimed at improving understanding of the mortgage market; and the second aimed at boosting financial empowerment for certain low- and moderate-income consumers.</p>\n'), ('ics', {'dtstamp': u'2015-12-10T14:20:29-0500', 'uid': u'/event/consumer-advisory-board-meeting-in-itta-bena-miss/', 'summary': u'Consumer Advisory Board meeting in Itta Bena, Miss.', 'dtend': u'2013-09-19T17:30:00-05:00', 'dtstart': u'2013-09-18T10:00:00-05:00', 'starting_tzinfo': u'America/Chicago', 'ending_tzinfo': u'America/Chicago'}), (u'id', 44119), (u'modified', u'2016-02-22T14:48:59+0000'), ('open_graph', {}), (u'order', 0), (u'parent', 0), (u'relative_url', u'/event/consumer-advisory-board-meeting-in-itta-bena-miss/'), ('rsvp', {u'contact': [u'']}), (u'slug', u'consumer-advisory-board-meeting-in-itta-bena-miss'), (u'status', u'publish'), (u'tags', [u'Consumer Advisory Board', u'Financial Empowerment', u'HMDA', u'Mortgages']), (u'title', u'Consumer Advisory Board meeting in Itta Bena, Miss.'), (u'title_plain', u'Consumer Advisory Board meeting in Itta Bena, Miss.'), (u'type', u'event'), (u'url', u'http://www.consumerfinance.gov/event/consumer-advisory-board-meeting-in-itta-bena-miss/'), ('venue', {u'name': u'Mississippi Valley State University', u'address': {u'city': u'Itta Bena', u'state': u'MS', u'street': u'14000 Highway 82 West', u'zip_code': u' 38941'}})])
        output = eventpage.DataConverter().convert(imported_data)
        expected_output = {'agenda_items-4-value-speakers-0-value-name': u'CAB Member, <NAME>', 'agenda_items-7-value-speakers-0-value-name': u'Financial Empowerment Program Specialist, <NAME>', 'agenda_items-0-value-speakers-2-value-url': u'', 'agenda_items-1-value-speakers-3-value-name': u'Mortgage Data Assets Team Lead, <NAME>', 'agenda_items-0-value-speakers-1-deleted': u'', 'agenda_items-4-value-speakers-0-value-url': u'', 'agenda_items-7-value-start_dt': '2016-02-22 13:30', 'agenda_items-9-value-end_dt': '2016-02-22 17:15', 'sidefoot-0-value-view_more-text': 'View More', 'youtube_url': u'', 'agenda_items-0-value-speakers-2-deleted': u'', 'agenda_items-9-value-description': u'Policy Discussion (Closed Session)', 'agenda_items-3-value-speakers-count': 1, 'agenda_items-1-value-speakers-0-deleted': u'', 'agenda_items-6-value-speakers-0-deleted': u'', 'categories-INITIAL_FORMS': '0', 'agenda_items-8-value-speakers-3-value-url': u'', 'agenda_items-8-value-speakers-1-value-url': u'', 'agenda_items-2-deleted': u'', 'agenda_items-9-value-speakers-0-value-name': u'Director, <NAME>', 'agenda_items-4-value-speakers-0-order': 0, 'agenda_items-8-value-speakers-2-deleted': u'', 'agenda_items-9-type': 'item', 'agenda_items-7-order': 7, 'agenda_items-3-type': 'item', 'agenda_items-9-value-speakers-0-order': 0, 'agenda_items-0-value-speakers-0-order': 0, 'agenda_items-9-value-speakers-0-value-url': u'http://www.consumerfinance.gov/the-bureau/about-rich-cordray/', 'agenda_items-3-value-end_dt': '2016-02-22 15:30', 'agenda_items-4-value-speakers-1-value-url': u'', 'agenda_items-0-value-speakers-0-value-url': u'http://www.consumerfinance.gov/the-bureau/about-rich-cordray/', 'agenda_items-0-value-description': u'Welcome and meeting overview (Closed Session)', 'agenda_items-1-order': 1, 'date_published': '2013-09-18', 'agenda_items-6-value-speakers-0-value-url': u'', 'agenda_items-7-value-speakers-count': 1, 'sidefoot-0-value-view_more-url': '/', 'agenda_items-1-value-speakers-1-value-name': u'Deputy Director, <NAME>', 'agenda_items-6-type': 'item', 'venue_zip': u'', 'agenda_items-2-value-description': u'Working lunch: CAB meeting resumes closed session', 'agenda_items-0-value-speakers-count': 3, 'agenda_items-8-value-speakers-1-value-name': u'Committee Chair <NAME>', 'venue_suite': u'', 'categories-MAX_NUM_FORMS': '2', 'agenda_items-6-value-description': u'Home Mortgage Disclosure Act discussion (Closed Session)', 'agenda_items-7-value-speakers-0-deleted': u'', 'agenda_items-8-value-speakers-2-value-url': u'', 'agenda_items-8-value-speakers-2-order': 2, 'agenda_items-7-value-description': u'Your Money, Your Goals discussion (Closed Session)', 'agenda_items-9-value-speakers-count': 2, 'start_dt': '2013-09-18 10:00', 'tags': u'"Consumer Advisory Board", "Financial Empowerment", HMDA, Mortgages', 'agenda_items-9-value-speakers-0-deleted': u'', 'agenda_items-8-type': 'item', 'agenda_items-7-value-speakers-0-value-url': u'', 'agenda_items-1-value-speakers-3-value-url': u'', 'agenda_items-2-value-speakers-0-value-url': u'', 'agenda_items-0-value-end_dt': '2016-02-22 11:00', 'body': u'<p>The\xa0CAB met with Director <NAME> to announce two initiatives: the first\xa0aimed at improving understanding of the mortgage market; and the second aimed at boosting financial empowerment for certain low- and moderate-income consumers.</p>\n', 'agenda_items-6-value-start_dt': '2016-02-22 10:30', 'agenda_items-7-value-location': u'Hilton Garden Inn', 'agenda_items-1-type': 'item', 'categories-MIN_NUM_FORMS': '0', 'agenda_items-3-value-speakers-0-value-name': u'CAB Chair, <NAME>\xf1onez', 'agenda_items-1-value-description': u'*CAB Public Session: Access to Credit, Information and Financial Resources (Open Session)', 'agenda_items-8-value-description': u'Committee Reports (Closed Session)', 'agenda_items-8-deleted': u'', 'agenda_items-0-value-speakers-1-value-name': u'CAB Staff Director, <NAME>', 'agenda_items-2-value-speakers-0-deleted': u'', 'sidefoot-0-value-relate_events': 'on', 'agenda_items-1-value-speakers-2-value-url': u'', 'agenda_items-9-order': 9, 'venue_state': u'MS', 'sidefoot-0-order': '0', 'agenda_items-3-value-description': u'Year in Review (Closed Session)', 'venue_name': u'Mississippi Valley State University', 'agenda_items-5-value-speakers-count': 0, 'agenda_items-4-type': 'item', 'agenda_items-9-deleted': u'', 'agenda_items-3-value-start_dt': '2016-02-22 14:30', 'agenda_items-8-value-end_dt': '2016-02-22 15:45', 'agenda_items-2-value-speakers-count': 1, 'agenda_items-2-value-speakers-0-order': 0, 'agenda_items-3-value-location': u'Social Science Auditorium ', 'agenda_items-8-value-speakers-2-value-name': u'Committee Chair <NAME>', 'agenda_items-2-type': 'item', 'agenda_items-7-value-speakers-0-order': 0, 'sidefoot-count': '1', 'agenda_items-5-value-start_dt': '2016-02-22 08:30', 'agenda_items-0-value-speakers-0-deleted': u'', 'agenda_items-6-value-end_dt': '2016-02-22 11:30', 'sidefoot-0-value-relate_posts': 'on', 'agenda_items-8-value-speakers-0-order': 0, 'agenda_items-1-deleted': u'', 'agenda_items-4-value-speakers-1-value-name': u'CAB Member, <NAME>', 'agenda_items-2-value-end_dt': '2016-02-22 14:30', 'flickr_url': u'', 'agenda_items-count': 10, 'agenda_items-4-value-speakers-count': 2, 'agenda_items-4-value-start_dt': '2016-02-22 15:30', 'agenda_items-9-value-speakers-1-order': 1, 'agenda_items-8-value-speakers-count': 4, 'agenda_items-3-deleted': u'', 'agenda_items-9-value-speakers-1-value-url': u'', 'agenda_items-5-value-description': u'CAB Committee meetings (Closed Session)\r\n', 'agenda_items-9-value-start_dt': '2016-02-22 16:00', 'agenda_items-1-value-speakers-0-order': 0, 'agenda_items-4-value-speakers-1-deleted': u'', 'categories-TOTAL_FORMS': '0', 'authors': u'"<NAME>"', 'agenda_items-5-order': 5, 'sidefoot-0-deleted': '', 'agenda_items-1-value-speakers-2-value-name': u'Assistant Director, Office of Financial Empowerment, <NAME>', 'agenda_items-7-value-end_dt': '2016-02-22 14:15', 'agenda_items-5-value-location': u'Hilton Garden Inn\r\n235 West Capitol Street\r\nJackson, MS 39201', 'agenda_items-8-value-speakers-3-value-name': u'Committee Chair <NAME>', 'agenda_items-0-value-speakers-1-value-url': u'', 'agenda_items-1-value-speakers-1-deleted': u'', 'agenda_items-0-value-start_dt': '2016-02-22 10:00', 'agenda_items-4-deleted': u'', 'agenda_items-0-value-speakers-2-value-name': u'CAB Chair, <NAME>\xf1onez', 'agenda_items-3-value-speakers-0-deleted': u'', 'agenda_items-0-value-location': u'Social Science Auditorium', 'agenda_items-7-deleted': u'', 'agenda_items-8-value-location': u'Hilton Garden Inn', 'agenda_items-1-value-speakers-1-order': 1, 'agenda_items-8-value-speakers-1-order': 1, 'agenda_items-1-value-speakers-3-order': 3, 'agenda_items-6-value-speakers-0-order': 0, 'agenda_items-1-value-speakers-3-deleted': u'', 'agenda_items-5-deleted': u'', 'agenda_items-2-value-start_dt': '2016-02-22 13:00', 'agenda_items-2-value-location': u'Social Science Auditorium ', 'agenda_items-8-value-speakers-3-deleted': u'', 'agenda_items-4-value-location': u'Social Science Auditorium', 'agenda_items-6-deleted': u'', 'agenda_items-0-type': 'item', 'agenda_items-1-value-speakers-count': 4, 'agenda_items-0-deleted': u'', 'agenda_items-8-value-speakers-0-deleted': u'', 'venue_city': u'Itta Bena', 'agenda_items-1-value-speakers-0-value-name': u'Director, <NAME>', 'agenda_items-1-value-start_dt': '2016-02-22 11:00', 'agenda_items-1-value-end_dt': '2016-02-22 13:00', 'venue_street': u'14000 Highway 82 West', 'sidefoot-0-value-limit': '3', 'agenda_items-1-value-speakers-2-order': 2, 'title': u'Consumer Advisory Board meeting in Itta Bena, Miss.', 'sidefoot-0-type': 'related_posts', 'agenda_items-4-value-speakers-0-deleted': u'', 'agenda_items-0-value-speakers-2-order': 2, 'sidefoot-0-value-relate_newsroom': 'on', 'agenda_items-0-order': 0, 'agenda_items-4-value-end_dt': '2016-02-22 17:30', 'agenda_items-2-order': 2, 'agenda_items-9-value-speakers-1-deleted': u'', 'agenda_items-1-value-location': u'Social Science Auditorium ', 'agenda_items-0-value-speakers-1-order': 1, 'archive_body': u'<p>The\xa0CAB met with Director <NAME> to announce two initiatives: the first\xa0aimed at improving understanding of the mortgage market; and the second aimed at boosting financial empowerment for certain low- and moderate-income consumers.</p>', 'agenda_items-3-order': 3, 'agenda_items-6-order': 6, 'agenda_items-6-value-location': u'Hilton Garden Inn', 'agenda_items-8-value-speakers-3-order': 3, 'agenda_items-2-value-speakers-0-value-name': u'CAB Chair, <NAME>', 'agenda_items-4-value-description': u'Policy Discussion: Trends and Themes in the field (Closed Session)', 'agenda_items-9-value-location': u'Hilton Garden Inn', 'agenda_items-6-value-speakers-count': 1, 'agenda_items-4-order': 4, 'slug': u'consumer-advisory-board-meeting-in-itta-bena-miss', 'agenda_items-9-value-speakers-1-value-name': u'Associate Director, Research, Markets and Regulations Division, <NAME>', 'agenda_items-8-value-speakers-0-value-url': u'', 'agenda_items-4-value-speakers-1-order': 1, 'agenda_items-1-value-speakers-0-value-url': u'http://www.consumerfinance.gov/the-bureau/about-rich-cordray/', 'agenda_items-1-value-speakers-1-value-url': u'', 'agenda_items-8-value-speakers-1-deleted': u'', 'agenda_items-8-value-start_dt': '2016-02-22 14:15', 'end_dt': '2013-09-19 17:30', 'agenda_items-5-type': 'item', 'agenda_items-0-value-speakers-0-value-name': u'Director, <NAME>', 'agenda_items-1-value-speakers-2-deleted': u'', 'agenda_items-3-value-speakers-0-order': 0, 'agenda_items-8-order': 8, 'agenda_items-8-value-speakers-0-value-name': u'Committee Chair <NAME>', 'agenda_items-5-value-end_dt': '2016-02-22 22:30', 'agenda_items-3-value-speakers-0-value-url': u'', 'agenda_items-6-value-speakers-0-value-name': u'Mortgage Data Assets Team Lead, <NAME>', 'agenda_items-7-type': 'item'}
        self.assertEqual(output, expected_output)


if __name__ == '__main__':
    unittest.main()

{'fraction_non_alphanumeric': 0.20307438923963766, 'fraction_numerical': 0.06255833104584134, 'mean_word_length': 6.47149302707137, 'pattern_counts': {'":': 0, '<': 120, '<?xml version=': 0, '>': 120, 'https://': 0, 'lorem ipsum': 0, 'www.': 16, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3691132', 'n_tokens_mistral': 14942, 'n_tokens_neox': 13692, 'n_words': 2618}