# Path: collectors/pfizer/parser.py
# def parse_record(res):
# 
#     # Init data
#     data = {}
# 
#     # Description
# 
#     key = 'study_type'
#     path = '.field-name-field-study-type .field-item::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     key = 'organization_id'
#     path = '.field-name-field-organization-id .field-item::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     key = 'nct_id'
#     path = '.field-name-field-clinical-trial-id .field-item::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     key = 'status'
#     path = '//label[text() = "Status"]/../text()'
#     value = ''.join(res.xpath(path).extract()).strip()
#     data[key] = value
# 
#     key = 'study_start_date'
#     path = '.field-name-field-study-start-date .field-item span::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     key = 'study_end_date'
#     path = '.field-name-field-study-end-date .field-item span::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     # Eligibility
# 
#     key = 'eligibility_criteria'
#     path = '.field-name-field-criteria .field-item *::text'
#     value = ''.join(res.css(path).extract())
#     data[key] = value
# 
#     key = 'gender'
#     path = '.field-name-field-gender .field-item::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     key = 'age_range'
#     path = '//label[text() = "Age Range:"]/../text()'
#     value = ''.join(res.xpath(path).extract()).strip()
#     data[key] = value
# 
#     key = 'healthy_volunteers_allowed'
#     path = '.field-name-field-healthy-volunteers-allowed .field-item::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     # Create record
#     record = Record.create(res.url, data)
# 
#     return record


from scrapy.spiders import Rule
from scrapy.spiders import CrawlSpider
from scrapy.linkextractors import LinkExtractor
from .parser import parse_record
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals



# Module API

class Spider(CrawlSpider):

    # Public

    name = 'pfizer'
    allowed_domains = ['pfizer.com']

    def __init__(self, conf=None, conn=None):

        # Save conf/conn
        self.conf = conf
        self.conn = conn

        # Make urls
        self.start_urls = [
            'http://www.pfizer.com/research/clinical_trials/find_a_trial?recr=0',
        ]

        # Make rules
        self.rules = [
            Rule(LinkExtractor(
                allow=r'find_a_trial/NCT\d+',

# Path: collectors/pfizer/parser.py
# def parse_record(res):
# 
#     # Init data
#     data = {}
# 
#     # Description
# 
#     key = 'study_type'
#     path = '.field-name-field-study-type .field-item::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     key = 'organization_id'
#     path = '.field-name-field-organization-id .field-item::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     key = 'nct_id'
#     path = '.field-name-field-clinical-trial-id .field-item::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     key = 'status'
#     path = '//label[text() = "Status"]/../text()'
#     value = ''.join(res.xpath(path).extract()).strip()
#     data[key] = value
# 
#     key = 'study_start_date'
#     path = '.field-name-field-study-start-date .field-item span::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     key = 'study_end_date'
#     path = '.field-name-field-study-end-date .field-item span::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     # Eligibility
# 
#     key = 'eligibility_criteria'
#     path = '.field-name-field-criteria .field-item *::text'
#     value = ''.join(res.css(path).extract())
#     data[key] = value
# 
#     key = 'gender'
#     path = '.field-name-field-gender .field-item::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     key = 'age_range'
#     path = '//label[text() = "Age Range:"]/../text()'
#     value = ''.join(res.xpath(path).extract()).strip()
#     data[key] = value
# 
#     key = 'healthy_volunteers_allowed'
#     path = '.field-name-field-healthy-volunteers-allowed .field-item::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     # Create record
#     record = Record.create(res.url, data)
# 
#     return record


# Path: collectors/pfizer/spider.py
from scrapy.spiders import Rule
from scrapy.spiders import CrawlSpider
from scrapy.linkextractors import LinkExtractor
from .parser import parse_record

from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals



# Module API

class Spider(CrawlSpider):

    # Public

    name = 'pfizer'
    allowed_domains = ['pfizer.com']

    def __init__(self, conf=None, conn=None):

        # Save conf/conn
        self.conf = conf
        self.conn = conn

        # Make urls
        self.start_urls = [
            'http://www.pfizer.com/research/clinical_trials/find_a_trial?recr=0',
        ]

        # Make rules
        self.rules = [
            Rule(LinkExtractor(
                allow=r'find_a_trial/NCT\d+',

            ), callback=parse_record),