# Path: collectors/isrctn/parser.py
# def parse_record(res):
# 
#     # Init data
#     data = {}
# 
#     # General
# 
#     key = 'isrctn_id'
#     path = '.ComplexTitle_primary::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     key = 'doi_isrctn_id'
#     path = '.ComplexTitle_secondary::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     key = 'title'
#     path = '//h1/text()'
#     value = res.xpath(path).extract_first()
#     data[key] = value
# 
#     kpath = '.Meta_name'
#     vpath = '.Meta_name+.Meta_value'
#     subdata = _parse_dict(res, kpath, vpath)
#     data.update(subdata)
# 
#     tag = 'h3'
#     text = 'Plain English Summary'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     section = _select_parent(res, tag, text)
#     subdata = _parse_dict(section, kpath, vpath)
#     data.update(subdata)
# 
#     # Contact information
# 
#     key = 'contacts'
#     tag = 'h2'
#     text = 'Contact information'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     first = 'type'
#     section = _select_parent(res, tag, text)
#     value = _parse_list(section, kpath, vpath, first)
#     data.update({key: value})
# 
#     # Additional identifiers
# 
#     tag = 'h2'
#     text = 'Additional identifiers'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     section = _select_parent(res, tag, text)
#     subdata = _parse_dict(section, kpath, vpath)
#     data.update(subdata)
# 
#     # Study information
# 
#     tag = 'h2'
#     text = 'Study information'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     section = _select_parent(res, tag, text)
#     subdata = _parse_dict(section, kpath, vpath)
#     data.update(subdata)
# 
#     # Eligibility
# 
#     tag = 'h2'
#     text = 'Eligibility'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     section = _select_parent(res, tag, text)
#     subdata = _parse_dict(section, kpath, vpath)
#     data.update(subdata)
# 
#     # Locations
# 
#     tag = 'h2'
#     text = 'Locations'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     section = _select_parent(res, tag, text)
#     subdata = _parse_dict(section, kpath, vpath)
#     data.update(subdata)
# 
#     # Sponsor information
# 
#     key = 'sponsors'
#     tag = 'h2'
#     text = 'Sponsor information'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     first = 'organisation'
#     section = _select_parent(res, tag, text)
#     value = _parse_list(section, kpath, vpath, first)
#     data.update({key: value})
# 
#     # Funders
# 
#     key = 'funders'
#     tag = 'h2'
#     text = 'Funders'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     first = 'funder_type'
#     section = _select_parent(res, tag, text)
#     value = _parse_list(section, kpath, vpath, first)
#     data.update({key: value})
# 
#     # Results and publications
# 
#     tag = 'h2'
#     text = 'Results and Publications'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     section = _select_parent(res, tag, text)
#     subdata = _parse_dict(section, kpath, vpath)
#     data.update(subdata)
# 
#     # Create record
#     record = Record.create(res.url, data)
# 
#     return record


from urllib import urlencode
from collections import OrderedDict
from datetime import date, timedelta
from scrapy.spiders import Rule
from scrapy.spiders import CrawlSpider
from scrapy.linkextractors import LinkExtractor
from .parser import parse_record
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals



# Module API

class Spider(CrawlSpider):

    # Public

    name = 'isrctn'
    allowed_domains = ['isrctn.com']

    def __init__(self, conf=None, conn=None, date_from=None, date_to=None):

        # Save conf/conn
        self.conf = conf
        self.conn = conn

        # Make start urls
        self.start_urls = _make_start_urls(
                prefix='http://www.isrctn.com/search',
                date_from=date_from, date_to=date_to)

        # Make rules
        self.rules = [
            Rule(LinkExtractor(
                allow=r'ISRCTN\d+',

# Path: collectors/isrctn/parser.py
# def parse_record(res):
# 
#     # Init data
#     data = {}
# 
#     # General
# 
#     key = 'isrctn_id'
#     path = '.ComplexTitle_primary::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     key = 'doi_isrctn_id'
#     path = '.ComplexTitle_secondary::text'
#     value = res.css(path).extract_first()
#     data[key] = value
# 
#     key = 'title'
#     path = '//h1/text()'
#     value = res.xpath(path).extract_first()
#     data[key] = value
# 
#     kpath = '.Meta_name'
#     vpath = '.Meta_name+.Meta_value'
#     subdata = _parse_dict(res, kpath, vpath)
#     data.update(subdata)
# 
#     tag = 'h3'
#     text = 'Plain English Summary'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     section = _select_parent(res, tag, text)
#     subdata = _parse_dict(section, kpath, vpath)
#     data.update(subdata)
# 
#     # Contact information
# 
#     key = 'contacts'
#     tag = 'h2'
#     text = 'Contact information'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     first = 'type'
#     section = _select_parent(res, tag, text)
#     value = _parse_list(section, kpath, vpath, first)
#     data.update({key: value})
# 
#     # Additional identifiers
# 
#     tag = 'h2'
#     text = 'Additional identifiers'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     section = _select_parent(res, tag, text)
#     subdata = _parse_dict(section, kpath, vpath)
#     data.update(subdata)
# 
#     # Study information
# 
#     tag = 'h2'
#     text = 'Study information'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     section = _select_parent(res, tag, text)
#     subdata = _parse_dict(section, kpath, vpath)
#     data.update(subdata)
# 
#     # Eligibility
# 
#     tag = 'h2'
#     text = 'Eligibility'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     section = _select_parent(res, tag, text)
#     subdata = _parse_dict(section, kpath, vpath)
#     data.update(subdata)
# 
#     # Locations
# 
#     tag = 'h2'
#     text = 'Locations'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     section = _select_parent(res, tag, text)
#     subdata = _parse_dict(section, kpath, vpath)
#     data.update(subdata)
# 
#     # Sponsor information
# 
#     key = 'sponsors'
#     tag = 'h2'
#     text = 'Sponsor information'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     first = 'organisation'
#     section = _select_parent(res, tag, text)
#     value = _parse_list(section, kpath, vpath, first)
#     data.update({key: value})
# 
#     # Funders
# 
#     key = 'funders'
#     tag = 'h2'
#     text = 'Funders'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     first = 'funder_type'
#     section = _select_parent(res, tag, text)
#     value = _parse_list(section, kpath, vpath, first)
#     data.update({key: value})
# 
#     # Results and publications
# 
#     tag = 'h2'
#     text = 'Results and Publications'
#     kpath = '.Info_section_title'
#     vpath = '.Info_section_title+p'
#     section = _select_parent(res, tag, text)
#     subdata = _parse_dict(section, kpath, vpath)
#     data.update(subdata)
# 
#     # Create record
#     record = Record.create(res.url, data)
# 
#     return record


# Path: collectors/isrctn/spider.py
from urllib import urlencode
from collections import OrderedDict
from datetime import date, timedelta
from scrapy.spiders import Rule
from scrapy.spiders import CrawlSpider
from scrapy.linkextractors import LinkExtractor
from .parser import parse_record

from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals



# Module API

class Spider(CrawlSpider):

    # Public

    name = 'isrctn'
    allowed_domains = ['isrctn.com']

    def __init__(self, conf=None, conn=None, date_from=None, date_to=None):

        # Save conf/conn
        self.conf = conf
        self.conn = conn

        # Make start urls
        self.start_urls = _make_start_urls(
                prefix='http://www.isrctn.com/search',
                date_from=date_from, date_to=date_to)

        # Make rules
        self.rules = [
            Rule(LinkExtractor(
                allow=r'ISRCTN\d+',

            ), callback=parse_record),