# Path: collectors/icdcm/record.py
# class Record(base.Record):
# 
#     # Config
# 
#     table = 'icdcm'
# 
#     # General
# 
#     name = Text(primary_key=True)
#     desc = Text()
#     terms = Array()
#     version = Text()
#     last_updated = Date('%Y-%m-%d')


import io
import logging
import zipfile
import requests
from scrapy.http import TextResponse
from .record import Record

    # For more information see:
    # https://www.cms.gov/Medicare/Coding/ICD10/2016-ICD-10-CM-and-GEMs.html
    URL = 'https://www.cms.gov/Medicare/Coding/ICD10/Downloads/2016-CM-Code-Tables-and-Index.zip'
    FILE = 'Tabular.xml'
    VERSION = 'ICD-10-CM'
    LAST_UPDATED = '2015-10-01'

    # Prepare xml
    zip = requests.get(URL).content
    xml = zipfile.ZipFile(io.BytesIO(zip)).open(FILE).read()
    res = TextResponse(url=URL, body=xml, encoding='utf-8')

    count = 0
    for diag in res.xpath('//diag'):
        # We need only leafs
        childs = diag.xpath('./diag')
        if not childs:
            continue

        # Get data
        data = {
            'name': diag.xpath('./name/text()').extract_first(),
            'desc': diag.xpath('./desc/text()').extract_first(),
            'terms': diag.xpath('.//note/text()').extract(),
            'version': VERSION,
            'last_updated': LAST_UPDATED,
        }

        # Create record

# Path: collectors/icdcm/record.py
# class Record(base.Record):
# 
#     # Config
# 
#     table = 'icdcm'
# 
#     # General
# 
#     name = Text(primary_key=True)
#     desc = Text()
#     terms = Array()
#     version = Text()
#     last_updated = Date('%Y-%m-%d')


# Path: collectors/icdcm/collector.py
import io
import logging
import zipfile
import requests
from scrapy.http import TextResponse
from .record import Record


    # For more information see:
    # https://www.cms.gov/Medicare/Coding/ICD10/2016-ICD-10-CM-and-GEMs.html
    URL = 'https://www.cms.gov/Medicare/Coding/ICD10/Downloads/2016-CM-Code-Tables-and-Index.zip'
    FILE = 'Tabular.xml'
    VERSION = 'ICD-10-CM'
    LAST_UPDATED = '2015-10-01'

    # Prepare xml
    zip = requests.get(URL).content
    xml = zipfile.ZipFile(io.BytesIO(zip)).open(FILE).read()
    res = TextResponse(url=URL, body=xml, encoding='utf-8')

    count = 0
    for diag in res.xpath('//diag'):
        # We need only leafs
        childs = diag.xpath('./diag')
        if not childs:
            continue

        # Get data
        data = {
            'name': diag.xpath('./name/text()').extract_first(),
            'desc': diag.xpath('./desc/text()').extract_first(),
            'terms': diag.xpath('.//note/text()').extract(),
            'version': VERSION,
            'last_updated': LAST_UPDATED,
        }

        # Create record

        record = Record.create(URL, data)