# Path: collectors/jprn/parser.py
# def parse_record(res):
#     fields_to_remove = [
#         'item',
#     ]
# 
#     # Parse rawdata
#     data = {}
# 
#     # Get meta
#     subdata = _parse_table(res, key_index=0, value_index=2)
#     data.update(subdata)
# 
#     # Process rawdata
#     rawdata = _parse_table(res, key_index=0, value_index=1)
#     prefix = ''
#     for key, value in rawdata.items():
# 
#         # Interventions
# 
#         newkey = 'interventions'
#         oldkey = 'interventionscontrol'
#         data.setdefault(newkey, [])
#         if key.startswith(oldkey):
#             data[newkey].append(value)
#             continue
# 
#         # Research contact person
# 
#         if key == 'name_of_lead_principal_investigator':
#             prefix = 'research_'
# 
#         # Public contact
# 
#         if key == 'name_of_contact_person':
#             prefix = 'public_'
# 
#         # Sponsor
# 
#         if key == 'name_of_primary_sponsor':
#             prefix = ''
# 
#         # Collect plain values
#         key = prefix + key
#         data[key] = value
# 
#     # Remove data
#     for key in fields_to_remove:
#         if key in data:
#             del data[key]
# 
#     identifier = data.get('unique_id_issued_by_umin')
#     data['unique_trial_number'] = data.get('unique_trial_number', identifier)
# 
#     # Create record
#     record = Record.create(res.url, data)
# 
#     return record


from functools import partial
from urllib import urlencode
from collections import OrderedDict
from scrapy.spiders import Rule
from scrapy.spiders import CrawlSpider
from scrapy.linkextractors import LinkExtractor
from six.moves.urllib.parse import urlparse, parse_qs
from .parser import parse_record
# Module API

class Spider(CrawlSpider):

    # Public

    name = 'jprn'
    allowed_domains = ['upload.umin.ac.jp']

    def __init__(self, conf=None, conn=None, page_from=None, page_to=None):

        # Save conf/conn
        self.conf = conf
        self.conn = conn

        # Default values
        if page_from is None:
            page_from = '1'
        if page_to is None:
            page_to = '1'

        # Make start urls
        self.start_urls = _make_start_urls(
                prefix='https://upload.umin.ac.jp/cgi-open-bin/ctr_e/index.cgi',
                page_from=page_from)

        # Make rules
        self.rules = [
            Rule(LinkExtractor(
                allow=r'cgi-open-bin/ctr_e/ctr_view.cgi',

# Path: collectors/jprn/parser.py
# def parse_record(res):
#     fields_to_remove = [
#         'item',
#     ]
# 
#     # Parse rawdata
#     data = {}
# 
#     # Get meta
#     subdata = _parse_table(res, key_index=0, value_index=2)
#     data.update(subdata)
# 
#     # Process rawdata
#     rawdata = _parse_table(res, key_index=0, value_index=1)
#     prefix = ''
#     for key, value in rawdata.items():
# 
#         # Interventions
# 
#         newkey = 'interventions'
#         oldkey = 'interventionscontrol'
#         data.setdefault(newkey, [])
#         if key.startswith(oldkey):
#             data[newkey].append(value)
#             continue
# 
#         # Research contact person
# 
#         if key == 'name_of_lead_principal_investigator':
#             prefix = 'research_'
# 
#         # Public contact
# 
#         if key == 'name_of_contact_person':
#             prefix = 'public_'
# 
#         # Sponsor
# 
#         if key == 'name_of_primary_sponsor':
#             prefix = ''
# 
#         # Collect plain values
#         key = prefix + key
#         data[key] = value
# 
#     # Remove data
#     for key in fields_to_remove:
#         if key in data:
#             del data[key]
# 
#     identifier = data.get('unique_id_issued_by_umin')
#     data['unique_trial_number'] = data.get('unique_trial_number', identifier)
# 
#     # Create record
#     record = Record.create(res.url, data)
# 
#     return record


# Path: collectors/jprn/spider.py
from functools import partial
from urllib import urlencode
from collections import OrderedDict
from scrapy.spiders import Rule
from scrapy.spiders import CrawlSpider
from scrapy.linkextractors import LinkExtractor
from six.moves.urllib.parse import urlparse, parse_qs
from .parser import parse_record

# Module API

class Spider(CrawlSpider):

    # Public

    name = 'jprn'
    allowed_domains = ['upload.umin.ac.jp']

    def __init__(self, conf=None, conn=None, page_from=None, page_to=None):

        # Save conf/conn
        self.conf = conf
        self.conn = conn

        # Default values
        if page_from is None:
            page_from = '1'
        if page_to is None:
            page_to = '1'

        # Make start urls
        self.start_urls = _make_start_urls(
                prefix='https://upload.umin.ac.jp/cgi-open-bin/ctr_e/index.cgi',
                page_from=page_from)

        # Make rules
        self.rules = [
            Rule(LinkExtractor(
                allow=r'cgi-open-bin/ctr_e/ctr_view.cgi',

            ), callback=parse_record),