# Path: collectors/takeda/parser.py
# def parse_record(res):
# 
#     # Init data
#     data = {}
# 
#     # Parse rawdata
#     gpath = 'h1'
#     kpath = 'p.eyebrowbold'
#     vpath = 'p.eyebrowbold+*'
#     rawdata = _parse_data(res, gpath, kpath, vpath)
#     for group, key, value in rawdata:
# 
#         # General
# 
#         if key == 'compound':
#             value = value.split(',')
# 
#         # Recruitment
# 
#         if key == 'locations':
#             value = value.split(',')
# 
#         # Collect plain values
#         data[key] = value
# 
#     # Extract results URL
#     selector = '#results div a::attr(href)'
#     value = res.css(selector).extract_first()
#     if value:
#         url = urlparse.urljoin(res.url, value)
#         data['download_the_clinical_trial_summary'] = url
#     else:
#         try:
#             del data['download_the_clinical_trial_summary']
#         except KeyError:
#             pass
# 
#     # Create record
#     record = Record.create(res.url, data)
# 
#     return record


from scrapy.spiders import Rule
from scrapy.spiders import CrawlSpider
from scrapy.linkextractors import LinkExtractor
from .parser import parse_record
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals



# Module API

class Spider(CrawlSpider):

    # Public

    name = 'takeda'
    allowed_domains = ['takedaclinicaltrials.com']

    def __init__(self, conf=None, conn=None):

        # Save conf/conn
        self.conf = conf
        self.conn = conn

        # Make urls
        self.start_urls = [
            'http://www.takedaclinicaltrials.com/browse/?protocol_id=',
        ]

        # Make rules
        self.rules = [
            Rule(LinkExtractor(
                allow=r'browse/summary/',

# Path: collectors/takeda/parser.py
# def parse_record(res):
# 
#     # Init data
#     data = {}
# 
#     # Parse rawdata
#     gpath = 'h1'
#     kpath = 'p.eyebrowbold'
#     vpath = 'p.eyebrowbold+*'
#     rawdata = _parse_data(res, gpath, kpath, vpath)
#     for group, key, value in rawdata:
# 
#         # General
# 
#         if key == 'compound':
#             value = value.split(',')
# 
#         # Recruitment
# 
#         if key == 'locations':
#             value = value.split(',')
# 
#         # Collect plain values
#         data[key] = value
# 
#     # Extract results URL
#     selector = '#results div a::attr(href)'
#     value = res.css(selector).extract_first()
#     if value:
#         url = urlparse.urljoin(res.url, value)
#         data['download_the_clinical_trial_summary'] = url
#     else:
#         try:
#             del data['download_the_clinical_trial_summary']
#         except KeyError:
#             pass
# 
#     # Create record
#     record = Record.create(res.url, data)
# 
#     return record


# Path: collectors/takeda/spider.py
from scrapy.spiders import Rule
from scrapy.spiders import CrawlSpider
from scrapy.linkextractors import LinkExtractor
from .parser import parse_record

from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals



# Module API

class Spider(CrawlSpider):

    # Public

    name = 'takeda'
    allowed_domains = ['takedaclinicaltrials.com']

    def __init__(self, conf=None, conn=None):

        # Save conf/conn
        self.conf = conf
        self.conn = conn

        # Make urls
        self.start_urls = [
            'http://www.takedaclinicaltrials.com/browse/?protocol_id=',
        ]

        # Make rules
        self.rules = [
            Rule(LinkExtractor(
                allow=r'browse/summary/',

            ), callback=parse_record),