# Path: fooltrader/connector/es_connector.py
# def df_to_es(df, doc_type, index_name=None, timestamp_filed='timestamp', security_item=None, force=False):
# def security_meta_to_es(security_type='stock'):
# def kdata_to_es(security_type='stock', start_code=None, end_code=None, force=False):
# def finance_sheet_to_es(sheet_type=None, start_code=None, end_code=None, force=False):
# def usa_stock_finance_to_es(force=False):
# def finance_event_to_es(event_type='finance_forecast', start_code=None, end_code=None, force=False):
# 
# Path: fooltrader/datamanager/china_stock_manager.py
# def crawl_stock_meta():
#     # 更新股票列表
#     # TODO:看是否有必要判断有新股上市，目前每天抓一次列表，问题不大
#     if True:
#         logger.info('download stock list start')
#         process_crawl(ChinaStockListSpider, {})
# 
#         # FIXME:better way to handle
#         init_env()
#         logger.info('download stock list finish')
# 
# def craw_stock_category():
#     process_crawl(SinaCategorySpider, {'category_type': 'sinaIndustry'})
#     process_crawl(SinaCategorySpider, {'category_type': 'sinaConcept'})
#     process_crawl(SinaCategorySpider, {'category_type': 'sinaArea'})
# 
# Path: fooltrader/utils/utils.py
# def init_process_log(file_name, log_dir=None):
#     root_logger = logging.getLogger()
# 
#     # reset the handlers
#     root_logger.handlers = []
# 
#     root_logger.setLevel(logging.INFO)
# 
#     if log_dir:
#         file_name = os.path.join(log_dir, file_name)
# 
#     fh = RotatingFileHandler(file_name, maxBytes=524288000, backupCount=10)
# 
#     fh.setLevel(logging.INFO)
# 
#     ch = logging.StreamHandler()
#     ch.setLevel(logging.INFO)
# 
#     # create formatter and add it to the handlers
#     formatter = logging.Formatter(
#         "%(levelname)s  %(threadName)s  %(asctime)s  %(name)s:%(lineno)s  %(funcName)s  %(message)s")
#     fh.setFormatter(formatter)
#     ch.setFormatter(formatter)
# 
#     # filter
#     # fh.addFilter(logging.Filter('fooltrader'))
# 
#     # add the handlers to the logger
#     root_logger.addHandler(fh)
#     root_logger.addHandler(ch)


import logging
from apscheduler.schedulers.background import BackgroundScheduler
from fooltrader.connector import es_connector
from fooltrader.datamanager.china_stock_manager import crawl_stock_meta, craw_stock_category
from fooltrader.utils.utils import init_process_log
# -*- coding: utf-8 -*-




init_process_log('crawling_stock_meta.log')

logger = logging.getLogger(__name__)

sched = BackgroundScheduler()


@sched.scheduled_job('cron', hour=00, minute=00)
def scheduled_job1():
    crawl_stock_meta()


@sched.scheduled_job('cron', hour=00, minute=10)
def scheduled_job2():
    craw_stock_category()
    es_connector.security_meta_to_es()


if __name__ == '__main__':
    logger.info("start crawling stock meta")

    crawl_stock_meta()

# Path: fooltrader/connector/es_connector.py
# def df_to_es(df, doc_type, index_name=None, timestamp_filed='timestamp', security_item=None, force=False):
# def security_meta_to_es(security_type='stock'):
# def kdata_to_es(security_type='stock', start_code=None, end_code=None, force=False):
# def finance_sheet_to_es(sheet_type=None, start_code=None, end_code=None, force=False):
# def usa_stock_finance_to_es(force=False):
# def finance_event_to_es(event_type='finance_forecast', start_code=None, end_code=None, force=False):
# 
# Path: fooltrader/datamanager/china_stock_manager.py
# def crawl_stock_meta():
#     # 更新股票列表
#     # TODO:看是否有必要判断有新股上市，目前每天抓一次列表，问题不大
#     if True:
#         logger.info('download stock list start')
#         process_crawl(ChinaStockListSpider, {})
# 
#         # FIXME:better way to handle
#         init_env()
#         logger.info('download stock list finish')
# 
# def craw_stock_category():
#     process_crawl(SinaCategorySpider, {'category_type': 'sinaIndustry'})
#     process_crawl(SinaCategorySpider, {'category_type': 'sinaConcept'})
#     process_crawl(SinaCategorySpider, {'category_type': 'sinaArea'})
# 
# Path: fooltrader/utils/utils.py
# def init_process_log(file_name, log_dir=None):
#     root_logger = logging.getLogger()
# 
#     # reset the handlers
#     root_logger.handlers = []
# 
#     root_logger.setLevel(logging.INFO)
# 
#     if log_dir:
#         file_name = os.path.join(log_dir, file_name)
# 
#     fh = RotatingFileHandler(file_name, maxBytes=524288000, backupCount=10)
# 
#     fh.setLevel(logging.INFO)
# 
#     ch = logging.StreamHandler()
#     ch.setLevel(logging.INFO)
# 
#     # create formatter and add it to the handlers
#     formatter = logging.Formatter(
#         "%(levelname)s  %(threadName)s  %(asctime)s  %(name)s:%(lineno)s  %(funcName)s  %(message)s")
#     fh.setFormatter(formatter)
#     ch.setFormatter(formatter)
# 
#     # filter
#     # fh.addFilter(logging.Filter('fooltrader'))
# 
#     # add the handlers to the logger
#     root_logger.addHandler(fh)
#     root_logger.addHandler(ch)


# Path: fooltrader/sched/sched_stock_meta.py
import logging
from apscheduler.schedulers.background import BackgroundScheduler
from fooltrader.connector import es_connector
from fooltrader.datamanager.china_stock_manager import crawl_stock_meta, craw_stock_category
from fooltrader.utils.utils import init_process_log

# -*- coding: utf-8 -*-




init_process_log('crawling_stock_meta.log')

logger = logging.getLogger(__name__)

sched = BackgroundScheduler()


@sched.scheduled_job('cron', hour=00, minute=00)
def scheduled_job1():
    crawl_stock_meta()


@sched.scheduled_job('cron', hour=00, minute=10)
def scheduled_job2():
    craw_stock_category()
    es_connector.security_meta_to_es()


if __name__ == '__main__':
    logger.info("start crawling stock meta")

    crawl_stock_meta()

    craw_stock_category()