# Path: fooltrader/contract/files_contract.py
# def get_kdata_path(item, source=None, fuquan='bfq', year=None, quarter=None):
#     source = adjust_source(item, source)
#     if source == 'sina':
#         if not year and not quarter:
#             return os.path.join(get_kdata_dir(item, fuquan), 'dayk.csv')
#         else:
#             return os.path.join(get_kdata_dir(item, fuquan), '{}Q{}.csv'.format(year, quarter))
#     else:
#         return os.path.join(get_kdata_dir(item, fuquan), '{}_dayk.csv'.format(source))
# 
# Path: fooltrader/utils/utils.py
# def index_df_with_time(df, index='timestamp'):
#     df = df.set_index(df[index], drop=False)
#     df.index = pd.to_datetime(df.index)
#     df = df.sort_index()
#     return df
# 
# def to_time_str(the_time, time_fmt=TIME_FORMAT_DAY):
#     try:
#         if time_fmt == TIME_FORMAT_MICRO:
#             return to_timestamp(the_time).strftime(time_fmt)[0:-3]
#         else:
#             return to_timestamp(the_time).strftime(time_fmt)
#     except Exception as e:
#         return the_time
# 
# def to_float(str, default=None):
#     try:
#         return float(str.replace(',', ''))
#     except Exception as e:
#         return default


import pandas as pd
import scrapy
from scrapy import Request
from scrapy import Selector
from scrapy import signals
from fooltrader.contract.files_contract import get_kdata_path
from fooltrader.utils.utils import index_df_with_time, to_time_str, to_float
    def download_sp500_price(self, response):
        trs = response.xpath('//*[@id="datatable"]/tr').extract()

        price_jsons = []

        try:
            for tr in trs[1:]:
                tds = Selector(text=tr).xpath('//td//text()').extract()
                tds = [x.strip() for x in tds if x.strip()]

                price_jsons.append({"timestamp": to_time_str(tds[0]),
                                    "close": to_float(tds[1])})

            if price_jsons:
                self.df_close = self.df_close.append(price_jsons, ignore_index=True)
                self.df_close = index_df_with_time(self.df_close)
        except Exception as e:
            self.logger.exception('error when getting sp500 price url={} error={}'.format(response.url, e))

    def download_sp500_pe(self, response):
        trs = response.xpath('//*[@id="datatable"]/tr').extract()

        price_jsons = []

        try:
            for tr in trs[1:]:
                tds = Selector(text=tr).xpath('//td//text()').extract()
                tds = [x.strip() for x in tds if x.strip()]

                price_jsons.append({"timestamp": to_time_str(tds[0]),

# Path: fooltrader/contract/files_contract.py
# def get_kdata_path(item, source=None, fuquan='bfq', year=None, quarter=None):
#     source = adjust_source(item, source)
#     if source == 'sina':
#         if not year and not quarter:
#             return os.path.join(get_kdata_dir(item, fuquan), 'dayk.csv')
#         else:
#             return os.path.join(get_kdata_dir(item, fuquan), '{}Q{}.csv'.format(year, quarter))
#     else:
#         return os.path.join(get_kdata_dir(item, fuquan), '{}_dayk.csv'.format(source))
# 
# Path: fooltrader/utils/utils.py
# def index_df_with_time(df, index='timestamp'):
#     df = df.set_index(df[index], drop=False)
#     df.index = pd.to_datetime(df.index)
#     df = df.sort_index()
#     return df
# 
# def to_time_str(the_time, time_fmt=TIME_FORMAT_DAY):
#     try:
#         if time_fmt == TIME_FORMAT_MICRO:
#             return to_timestamp(the_time).strftime(time_fmt)[0:-3]
#         else:
#             return to_timestamp(the_time).strftime(time_fmt)
#     except Exception as e:
#         return the_time
# 
# def to_float(str, default=None):
#     try:
#         return float(str.replace(',', ''))
#     except Exception as e:
#         return default


# Path: fooltrader/spiders/america/sp500_spider.py
import pandas as pd
import scrapy
from scrapy import Request
from scrapy import Selector
from scrapy import signals
from fooltrader.contract.files_contract import get_kdata_path
from fooltrader.utils.utils import index_df_with_time, to_time_str, to_float

    def download_sp500_price(self, response):
        trs = response.xpath('//*[@id="datatable"]/tr').extract()

        price_jsons = []

        try:
            for tr in trs[1:]:
                tds = Selector(text=tr).xpath('//td//text()').extract()
                tds = [x.strip() for x in tds if x.strip()]

                price_jsons.append({"timestamp": to_time_str(tds[0]),
                                    "close": to_float(tds[1])})

            if price_jsons:
                self.df_close = self.df_close.append(price_jsons, ignore_index=True)
                self.df_close = index_df_with_time(self.df_close)
        except Exception as e:
            self.logger.exception('error when getting sp500 price url={} error={}'.format(response.url, e))

    def download_sp500_pe(self, response):
        trs = response.xpath('//*[@id="datatable"]/tr').extract()

        price_jsons = []

        try:
            for tr in trs[1:]:
                tds = Selector(text=tr).xpath('//td//text()').extract()
                tds = [x.strip() for x in tds if x.strip()]

                price_jsons.append({"timestamp": to_time_str(tds[0]),

                                    "pe": to_float(tds[1])})