<jupyter_start><jupyter_text>券商分點這個券商分點可以繞過「圖片認證碼」請大家再三保密使用XD不然哪天就失效了，抓取券商分點，每檔股票總共可以分成三個步驟：1. 開啟驗證碼網頁，得到驗證碼2. 利用驗證碼來得到券商分點3. 整理券商分點資訊<jupyter_code># ---------------
# 偽裝成瀏覽器
# ---------------
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0.1) Gecko/2010010' \
    '1 Firefox/4.0.1',
    'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language':'en-us,en;q=0.5',
    'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.7'}<jupyter_output><empty_output><jupyter_text>1. 開啟驗證碼網頁，得到驗證碼這邊的驗證碼不是「圖片驗證碼喔！」而是 `viewstate` 跟 `eventvalidation`<jupyter_code># ---------------
# 開啟有驗證碼的網頁
# ---------------
import requests
rs = requests.session()

def get_verify_code(rs):
    res = rs.get( 'http://bsr.twse.com.tw/bshtm/bsMenu.aspx', stream = True, verify = False, headers = headers, timeout=None )
    # ---------------
    # 從網頁中抓取一些關鍵字
    # ---------------
    import re
    
    # get view state
    viewstate = re.search( 'VIEWSTATE"\s+value=.*=', res.text )
    viewstate = viewstate.group()[18:]
    
    # get eventvalidation
    eventvalidation = re.search( 'EVENTVALIDATION"\s+value=.*\w', res.text )
    eventvalidation = eventvalidation.group( )[ 24: ]
    
    return viewstate, eventvalidation

viewstate, eventvalidation = get_verify_code(rs)

print('----')
print('viewstate', viewstate)
print('----')
print('eventvalidation',eventvalidation)
print('----')
ve<jupyter_output><empty_output><jupyter_text>2. 透過驗證碼爬取資料並存檔<jupyter_code># ---------------
# 爬取資料
# ---------------

def get_data(rs, stock_id, viewstate, eventvalidation):
    import time
    payload = {
    '__EVENTTARGET':'',
    '__EVENTARGUMENT':'',
    '__LASTFOCUS':'',
    '__VIEWSTATE' : viewstate,                      #encode_viewstate[:-1],
    '__EVENTVALIDATION' : eventvalidation,          #encode_eventvalidation[:-1],
    'RadioButton_Normal' : 'RadioButton_Normal',
    'TextBox_Stkno' : stock_id,
    'CaptchaControl1 ' : 'Z67YB',
    'btnOK' : '%E6%9F%A5%E8%A9%A2',
    }
    rs.post( "https://bsr.twse.com.tw/bshtm/bsMenu.aspx", data=payload, headers=headers, stream = True )
    time.sleep(1)
    res = rs.get( 'https://bsr.twse.com.tw/bshtm/bsContent.aspx',verify = False ,stream = True, )
    return res

res = get_data(rs, '1101', viewstate, eventvalidation)
res.encoding='big5'
open('test.html', 'w', encoding='utf-8').write(res.text)<jupyter_output>C:\Users\Dandy\Anaconda3\envs\finlab\lib\site-packages\urllib3\connectionpool.py:847: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
  InsecureRequestWarning)
C:\Users\Dandy\Anaconda3\envs\finlab\lib\site-packages\urllib3\connectionpool.py:847: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
  InsecureRequestWarning)<jupyter_text>3. 整理資料<jupyter_code>import pandas as pd
from io import StringIO

def parse_data(text):
    lines = text.split('\n')
    lines = [l for l in lines if len(l.split(',')) == 11]
    df = pd.read_csv(StringIO('\n'.join(lines)))

    first_df = df[df.columns[:5]]
    second_df = df[df.columns[6:]]
    second_df.columns = second_df.columns.str.replace('.1', '')
    final_df = first_df.append(second_df).set_index('序號').sort_index().dropna()
    return final_df




df = parse_data(res.text)<jupyter_output><empty_output><jupyter_text>完整下載範例（每天來按一下） 假如無法下載，可以參考http://finlabcourse.imotor.com/viewthread.php?tid=322&extra=page%3D1&page=3來做一些更動，就可以順利執行了～<jupyter_code>import time
import requests
from io import StringIO
import pandas as pd
import datetime

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0.1) Gecko/2010010' \
    '1 Firefox/4.0.1',
    'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language':'en-us,en;q=0.5',
    'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.7'}

def get_data(rs, stock_id, viewstate, eventvalidation):
    import time
    payload = {
    '__EVENTTARGET':'',
    '__EVENTARGUMENT':'',
    '__LASTFOCUS':'',
    '__VIEWSTATE' : viewstate,                      #encode_viewstate[:-1],
    '__EVENTVALIDATION' : eventvalidation,          #encode_eventvalidation[:-1],
    'RadioButton_Normal' : 'RadioButton_Normal',
    'TextBox_Stkno' : stock_id,
    'CaptchaControl1 ' : 'Z67YB',
    'btnOK' : '%E6%9F%A5%E8%A9%A2',
    }

    rs.post( "https://bsr.twse.com.tw/bshtm/bsMenu.aspx", data=payload, headers=headers)#, verify = False, stream = True )
    time.sleep(1)
    res = rs.get( 'https://bsr.twse.com.tw/bshtm/bsContent.aspx')#,verify = False ,stream = True, )
    return res

def get_verify_code(rs):
    res = rs.get( 'https://bsr.twse.com.tw/bshtm/bsMenu.aspx', stream = True, verify = False, headers = headers, timeout=None )
    # ---------------
    # 從網頁中抓取一些關鍵字
    # ---------------
    import re
    
    # get view state
    viewstate = re.search( 'VIEWSTATE"\s+value=.*=', res.text )
    viewstate = viewstate.group()[18:]
    
    # get eventvalidation
    eventvalidation = re.search( 'EVENTVALIDATION"\s+value=.*\w', res.text )
    eventvalidation = eventvalidation.group( )[ 24: ]
    
    return viewstate, eventvalidation

def download_stock(stock_id, path):
    rs = requests.Session()
    #viewstate, eventvalidation = get_verify_code(rs)
    
    while True:
        try:
            viewstate, eventvalidation = get_verify_code(rs)
            break # 抓資料成功,進行下一步
        except Exception as e:
            print(e) # 印出無法連線原因
            print('無法拿到資料, 等 31 sec') # server好像會擋30sec
            time.sleep(31)
            continue # 抓資料失敗,重新抓資料
    
    time.sleep(2)
    res = get_data(rs, stock_id, viewstate, eventvalidation)
    res.encoding='big5'
    open(path, 'w', encoding='utf-8').writelines(res.text)
    
def stock_list():
    res = requests.get('https://dts.twse.com.tw/opendata/t187ap03_L.csv')
    res.encoding='utf-8'
    df = pd.read_csv(StringIO(res.text), index_col=['公司代號'])
    return df

import os

# build main dir
main_dir = os.path.join('data', 'broker_history')
if not os.path.isdir(main_dir):
    os.mkdir(main_dir)
    
# build date dir
now = datetime.datetime.now()
date_dir = os.path.join(main_dir, now.strftime('%Y%m%d'))
if not os.path.isdir(date_dir):
    os.mkdir(date_dir)

    
slist = stock_list()
for s in slist.index:
    stock_path = os.path.join(date_dir, str(s) + '.csv')
    print(stock_path)
    if not os.path.isfile(stock_path):
        print('downloading...')
        download_stock(str(s), stock_path)
        time.sleep(10)<jupyter_output><empty_output><jupyter_text>券商總買賣<jupyter_code>buy = df['買進股數'].astype(int).groupby(df['券商']).sum()
sell= df['賣出股數'].astype(int).groupby(df['券商']).sum()
total = buy - sell
total<jupyter_output><empty_output><jupyter_text>主力買賣超<jupyter_code># 買方top 15
total.nlargest(15)
# 賣方 top 15
total.nsmallest(15)
# 主力買賣超
total.nlargest(15).sum() + total.nsmallest(15).sum()<jupyter_output><empty_output><jupyter_text>買賣家數差<jupyter_code># 買賣家數差
(total > 0).sum() - (total < 0).sum()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17778698715292168, 'fraction_numerical': 0.02348390661693604, 'mean_word_length': 3.5563247325361864, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '333200', 'license': 'MIT', 'n_tokens_mistral': 3064, 'n_tokens_neox': 3041, 'n_words': 411, 'path': 'finlab/brokers.ipynb', 'repo_name': 'DandyWei/finlab_course_ml', 'size': '7239'}