<commit_before># encoding: utf-8
# pylint: disable=E1101
# 融券 ＆＆借券賣出
# http://www.twse.com.tw/zh/page/trading/exchange/TWT93U.html


import scrapy
from scrapy import signals, Spider
from urllib.parse import urlencode
import time
from random import randint
import datetime
import logging
from copy import copy
from dateutil.relativedelta import relativedelta
import collections
import json
from bs4 import BeautifulSoup
from random import randint
import re
import csv
import traceback

# for cloud function call && scrapy crawl command call
# softlink package folder to root
try:
    from package.tools import is_settle, format_number, getDateObj
    from package.storage import Storage
except:
    from spiders.package.tools import is_settle, format_number, getDateObj
    from spiders.package.storage import Storage



class CreditSpider(scrapy.Spider):
    name = 'securities_loan_and_stock_lending'
        
    def __init__(self, category=None, *args, **kwargs):
        super(CreditSpider, self).__init__(*args, **kwargs)
        
        self.dataStorage = Storage(self.name)
        self.data = collections.OrderedDict()
        self.today = datetime.date.today()
        self.url = 'http://www.twse.com.tw/exchangeReport/TWT93U'
        self.params = {
            'response': 'csv',
            'date': '20190101'
        }
        self.startDate = getattr(self, 'start', self.getFormatDate(self.today))
        self.endDate = getattr(self, 'end', self.getFormatDate(self.today))
        self.startObj = getDateObj(self.startDate, None)
        self.endObj = getDateObj(self.endDate, None)

    def getFormatDate(self, date):
        year = str(date.year)
        month = str(date.month) if len(str(date.month)) != 1 else '0' + str(date.month)
        day = str(date.day) if len(str(date.day)) != 1 else '0' + str(date.day)
        return year + month + day

    def getQueryDate(self, date):
        year = str(date.year)
        month = str(date.month) if len(str(date.month)) != 1 else '0' + str(date.month)
        day = str(date.day) if len(str(date.day)) != 1 else '0' + str(date.day)
        return year + '/' + month+ '/' + day

    def start_requests(self):
        targetDateObj = copy(self.startObj)
        while(targetDateObj['datetime'] <= self.endObj['datetime']):
            self.params['date'] = self.getFormatDate(targetDateObj['datetime'])
            url = self.url + '?' + urlencode(self.params)
            time.sleep(randint(2,3))
            yield scrapy.Request(
                url=url,
                callback=self.parse,
                cb_kwargs=dict(targetDateObj=copy(targetDateObj), params=copy(self.params)),
                errback=self.handle_failure)

            targetDateObj['datetime'] = targetDateObj['datetime'] + relativedelta(days=1)
            targetDateObj['year'] = targetDateObj['datetime'].year
            targetDateObj['month'] = targetDateObj['datetime'].month
            targetDateObj['day'] = targetDateObj['datetime'].day

    def handle_failure(self, failure):
        self.log(failure, level=logging.ERROR)
        # try with a new proxy
        self.log('restart from the failed url {}'.format(failure.request.url))
        time.sleep(120)
        yield scrapy.Request(
            url=failure.request.url,
            callback=self.parse,
            cb_kwargs=failure.request.cb_kwargs,
            errback=self.handle_failure)

    def parse(self, response, targetDateObj, params):
        queryDate = self.getQueryDate(targetDateObj['datetime'])
        cr = csv.reader(response.text.splitlines(), delimiter=',')
        my_list = list(cr)

        if len(my_list) < 200:
            print(queryDate, "no data")
            return

        print(queryDate)

        for row in my_list:
            try:
                if len(row) != 16:
                    continue

                code = re.sub('=|"', '', row[0])

                if code == '':
                    continue

                credit_data = {
                    "date": queryDate,
                    "sl_preDay_balance": format_number(row[2]),
                    "sl_sell": format_number(row[3]),
                    "sl_buy": format_number(row[4]),
                    "sl_cash_stock": format_number(row[5]),
                    "sl_day_balance": format_number(row[6]),
                    "sl_limit": format_number(row[7]),
                    "bw_preDay_balance": format_number(row[8]),
                    "bw_sell_on_day": format_number(row[9]),
                    "bw_return_on_day": format_number(row[10]),
                    "bw_adjust_on_day": format_number(row[11]),
                    "bw_day_balance": format_number(row[12]),
                    "bw_limit_on_next_business_day": format_number(row[13])
                }
    
                if code not in self.data:
                    self.data[code] = {
                        "code": code,
                        "name": row[1],
                        "credit_data": [credit_data]
                    }
                else:
                    self.data[code]['credit_data'].append(credit_data)
            except:
                continue

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(CreditSpider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
        return spider

    def spider_closed(self, spider):
        # filename = 'test/sl.json'
        # with open(filename, 'w', encoding='utf8') as f:
        #     f.write(json.dumps(self.data, ensure_ascii=False, indent=2, sort_keys=True))
        fileName = 'sl'
        newData = dict()
        try:
            newData = self.dataStorage.getOldData(fileName)
        except:
            pass
        
        for code in self.data:
            if code not in newData:
                newData[code] = self.data[code]
            else:
                newData[code]['credit_data'] += self.data[code]['credit_data']

        for target_list in expression_list:
            pass
        self.dataStorage.saveData(fileName, newData)
<commit_msg>change the scheme  of the store object<commit_after># encoding: utf-8
# pylint: disable=E1101
# 融券 ＆＆借券賣出
# http://www.twse.com.tw/zh/page/trading/exchange/TWT93U.html


import scrapy
from scrapy import signals, Spider
from urllib.parse import urlencode
import time
from random import randint
import datetime
import logging
from copy import copy
from dateutil.relativedelta import relativedelta
import collections
import json
from bs4 import BeautifulSoup
from random import randint
import re
import csv
import traceback

# for cloud function call && scrapy crawl command call
# softlink package folder to root
try:
    from package.tools import is_settle, format_number, getDateObj
    from package.storage import Storage
except:
    from spiders.package.tools import is_settle, format_number, getDateObj
    from spiders.package.storage import Storage



class CreditSpider(scrapy.Spider):
    name = 'securities_loan_and_stock_lending'
        
    def __init__(self, category=None, *args, **kwargs):
        super(CreditSpider, self).__init__(*args, **kwargs)
        
        self.dataStorage = Storage(self.name)
        self.data = collections.OrderedDict()
        self.today = datetime.date.today()
        self.url = 'http://www.twse.com.tw/exchangeReport/TWT93U'
        self.params = {
            'response': 'csv',
            'date': '20190101'
        }
        self.startDate = getattr(self, 'start', self.getFormatDate(self.today))
        self.endDate = getattr(self, 'end', self.getFormatDate(self.today))
        self.startObj = getDateObj(self.startDate, None)
        self.endObj = getDateObj(self.endDate, None)

    def getFormatDate(self, date):
        year = str(date.year)
        month = str(date.month) if len(str(date.month)) != 1 else '0' + str(date.month)
        day = str(date.day) if len(str(date.day)) != 1 else '0' + str(date.day)
        return year + month + day

    def getQueryDate(self, date):
        year = str(date.year)
        month = str(date.month) if len(str(date.month)) != 1 else '0' + str(date.month)
        day = str(date.day) if len(str(date.day)) != 1 else '0' + str(date.day)
        return year + '/' + month+ '/' + day

    def start_requests(self):
        targetDateObj = copy(self.startObj)
        while(targetDateObj['datetime'] <= self.endObj['datetime']):
            self.params['date'] = self.getFormatDate(targetDateObj['datetime'])
            url = self.url + '?' + urlencode(self.params)
            time.sleep(randint(2,3))
            yield scrapy.Request(
                url=url,
                callback=self.parse,
                cb_kwargs=dict(targetDateObj=copy(targetDateObj), params=copy(self.params)),
                errback=self.handle_failure)

            targetDateObj['datetime'] = targetDateObj['datetime'] + relativedelta(days=1)
            targetDateObj['year'] = targetDateObj['datetime'].year
            targetDateObj['month'] = targetDateObj['datetime'].month
            targetDateObj['day'] = targetDateObj['datetime'].day

    def handle_failure(self, failure):
        self.log(failure, level=logging.ERROR)
        # try with a new proxy
        self.log('restart from the failed url {}'.format(failure.request.url))
        time.sleep(120)
        yield scrapy.Request(
            url=failure.request.url,
            callback=self.parse,
            cb_kwargs=failure.request.cb_kwargs,
            errback=self.handle_failure)

    def parse(self, response, targetDateObj, params):
        queryDate = self.getQueryDate(targetDateObj['datetime'])
        cr = csv.reader(response.text.splitlines(), delimiter=',')
        my_list = list(cr)

        if len(my_list) < 200:
            print(queryDate, "no data")
            return

        print(queryDate)

        for row in my_list:
            try:
                if len(row) != 16:
                    continue

                code = re.sub('=|"', '', row[0])

                if code == '':
                    continue

                credit_data = {
                    "date": queryDate,
                    "sl_preDay_balance": format_number(row[2]),
                    "sl_sell": format_number(row[3]),
                    "sl_buy": format_number(row[4]),
                    "sl_cash_stock": format_number(row[5]),
                    "sl_day_balance": format_number(row[6]),
                    "sl_limit": format_number(row[7]),
                    "bw_preDay_balance": format_number(row[8]),
                    "bw_sell_on_day": format_number(row[9]),
                    "bw_return_on_day": format_number(row[10]),
                    "bw_adjust_on_day": format_number(row[11]),
                    "bw_day_balance": format_number(row[12]),
                    "bw_limit_on_next_business_day": format_number(row[13])
                }
    
                if code not in self.data:
                    credit_data_item = {}
                    credit_data_item[queryDate] = credit_data
                    self.data[code] = {
                        "code": code,
                        "name": row[1],
                        "credit_data": credit_data_item
                    }
                else:
                    self.data[code]['credit_data'][queryDate] = credit_data
            except:
                traceback.print_exc()
                continue

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(CreditSpider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
        return spider

    def spider_closed(self, spider):
        fileName = 'sl'
        newData = dict()
        try:
            newData = self.dataStorage.getOldData(fileName)
        except:
            pass
        
        for code in self.data:
            if code not in newData:
                newData[code] = self.data[code]
            else:
                for date in self.data[code]['credit_data']:
                    item = self.data[code]['credit_data'][date]
                    if date in newData[code]['credit_data']:
                        newData[code]['credit_data'][date].update(item)
                    else:
                        newData[code]['credit_data'][date] = item

        self.dataStorage.saveData(fileName, newData)

{'fraction_non_alphanumeric': 0.11533840608730477, 'fraction_numerical': 0.008970764917901481, 'mean_word_length': 2.0768851651059634, 'pattern_counts': {'":': 32, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4553064', 'n_tokens_mistral': 3799, 'n_tokens_neox': 3586, 'n_words': 742}