# Path: collectors/fda_dap/record.py
# class Record(base.Record):
# 
#     # Config
# 
#     table = 'fda_dap'
# 
#     # General
# 
#     id = Text(primary_key=True)
#     drug_name = Text()
#     active_ingredients = Text()
#     company = Text()
#     fda_application_num = Text()
#     supplement_number = Integer()
#     action_date = Date('%m/%d/%Y')
#     approval_type = Text()
#     notes = Text()
#     documents = Json()


import urlparse
import re
import random
from scrapy.utils.url import canonicalize_url
from scrapy.spiders import CrawlSpider
from scrapy.http import Request
from scrapy.http.cookies import CookieJar
from .record import Record
        def _parse_links(selector):
            names = [_clean_document_name(name)
                     for name in selector.css('a::text').extract()]
            urls = [url.strip()
                    for url in selector.css('a::attr(href)').extract()]
            msg = 'The number of document names and URLs is different (%d != %d)' % (len(names), len(urls))
            assert len(names) == len(urls), msg

            return [{'name': name, 'urls': [url]}
                    for name, url in zip(names, urls)]

        def _parse_row(row, url, drug_details):
            columns = row.css('td')

            data = {
                'action_date': _extract_text(columns[0]),
                'supplement_number': _extract_text(columns[1]),
                'approval_type': _extract_text(columns[2]),
                'documents': _parse_links(columns[3]),
                'notes': _extract_text(columns[4]) or None,
            }

            intersection = set(drug_details.keys()).intersection(data.keys())
            assert not intersection, \
                'Drug details and data contain equal keys: %s' % intersection

            data.update(drug_details)
            data['id'] = '-'.join([data['fda_application_num'],
                                   data['supplement_number']])


# Path: collectors/fda_dap/record.py
# class Record(base.Record):
# 
#     # Config
# 
#     table = 'fda_dap'
# 
#     # General
# 
#     id = Text(primary_key=True)
#     drug_name = Text()
#     active_ingredients = Text()
#     company = Text()
#     fda_application_num = Text()
#     supplement_number = Integer()
#     action_date = Date('%m/%d/%Y')
#     approval_type = Text()
#     notes = Text()
#     documents = Json()


# Path: collectors/fda_dap/spider.py
import urlparse
import re
import random
from scrapy.utils.url import canonicalize_url
from scrapy.spiders import CrawlSpider
from scrapy.http import Request
from scrapy.http.cookies import CookieJar
from .record import Record

        def _parse_links(selector):
            names = [_clean_document_name(name)
                     for name in selector.css('a::text').extract()]
            urls = [url.strip()
                    for url in selector.css('a::attr(href)').extract()]
            msg = 'The number of document names and URLs is different (%d != %d)' % (len(names), len(urls))
            assert len(names) == len(urls), msg

            return [{'name': name, 'urls': [url]}
                    for name, url in zip(names, urls)]

        def _parse_row(row, url, drug_details):
            columns = row.css('td')

            data = {
                'action_date': _extract_text(columns[0]),
                'supplement_number': _extract_text(columns[1]),
                'approval_type': _extract_text(columns[2]),
                'documents': _parse_links(columns[3]),
                'notes': _extract_text(columns[4]) or None,
            }

            intersection = set(drug_details.keys()).intersection(data.keys())
            assert not intersection, \
                'Drug details and data contain equal keys: %s' % intersection

            data.update(drug_details)
            data['id'] = '-'.join([data['fda_application_num'],
                                   data['supplement_number']])


            return Record.create(None, data)