<jupyter_start><jupyter_text>以下複習bs4的使用<jupyter_code>num = 3000
PTT_stock_URL = 'https://www.ptt.cc/bbs/Stock/index'+str(num)+'.html'

driver = webdriver.PhantomJS(executable_path='/Users/mac/Desktop/Programming/phantomjs-2.1.1-macosx/bin/phantomjs')
driver.get(PTT_stock_URL)
PTT_page = driver.page_source

soup = BeautifulSoup(PTT_page, 'lxml')<jupyter_output><empty_output><jupyter_text>bs4裡面的method都是針對單一個對象的，不要多目標一起用會出錯<jupyter_code>soup.title
soup.title.string
print(soup.prettify())
a_tags = soup.find_all('a',  string=re.compile('三大')) #只搜尋子字串含有___的人
a_tags
article_list = soup.find_all(href=re.compile('/bbs/Stock/M')) #代表真實文章的所有列表
article_list
soup.find_all(href=re.compile('/bbs/Stock/M'))[0].string #只有單個品項才可以取出string
soup.find_all(href=re.compile('/bbs/Stock/M'))[0].attrs #只有單個品項才可以取出attrs
soup.find_all(href=re.compile('/bbs/Stock/M'))[0].attrs['href'] #單個品項先取出attrs再用字典取出href
div_tags = soup.find_all('div', class_='r-ent') #注意，是class_
div_tags
#日期
div_tags[0].find('div', class_='date').string
#作者
div_tags[0].find('div', class_='author').string
#推文噓文數目
div_tags[0].find('div', class_='nrec').string
#文章名稱、URL
div_tags[0].find('a', href=re.compile('/bbs/Stock/M'))
#文章URL
div_tags[0].find('a', href=re.compile('/bbs/Stock/M')).attrs['href']
#文章名稱
div_tags[0].find('a', href=re.compile('/bbs/Stock/M')).string<jupyter_output><empty_output><jupyter_text>總結以上一個文章列表可以獲得的資訊<jupyter_code>dict_page_topic = []
dict_page_topic_URL = []
dict_page_author = []
dict_page_date = []
dict_page_good_boo = []

for topic_content in soup.find_all('div', class_='r-ent'):
    dict_page_topic.append(topic_content.find('a', href=re.compile('/bbs/Stock/M')).string)
    dict_page_topic_URL.append('https://www.ptt.cc'+topic_content.find('a', href=re.compile('/bbs/Stock/M')).attrs['href'])
    dict_page_author.append(topic_content.find('div', class_='author').string)
    dict_page_date.append(topic_content.find('div', class_='date').string)
    dict_page_good_boo.append(topic_content.find('div', class_='nrec').string)
    
df = pd.DataFrame({
        'topic': dict_page_topic,
        'topic_URL': dict_page_topic_URL,
        'author': dict_page_author,
        'date': dict_page_date,
        'good_boo': dict_page_good_boo,
    })

df<jupyter_output><empty_output><jupyter_text>用for迴圈拓展爬3000-3718頁面的所有文章（近2年），並障礙排除刪除的文章<jupyter_code>dict_page_topic = []
dict_page_topic_URL = []
dict_page_author = []
dict_page_date = []
dict_page_good_boo = []

driver = webdriver.PhantomJS(executable_path='/Users/mac/Desktop/Programming/phantomjs-2.1.1-macosx/bin/phantomjs')

for i in range(3000,3718):
    PTT_stock_URL = 'https://www.ptt.cc/bbs/Stock/index'+str(i)+'.html'
    driver.get(PTT_stock_URL)
    PTT_page = driver.page_source
    soup = BeautifulSoup(PTT_page, 'lxml')
    
    for topic_content in soup.find_all('div', class_='r-ent'):
        
        try:
            topic_content.find('a', href=re.compile('/bbs/Stock/M')).string
            dict_page_topic.append(topic_content.find('a', href=re.compile('/bbs/Stock/M')).string)
            dict_page_topic_URL.append('https://www.ptt.cc'+topic_content.find('a', href=re.compile('/bbs/Stock/M')).attrs['href'])
            dict_page_author.append(topic_content.find('div', class_='author').string)
            dict_page_date.append(topic_content.find('div', class_='date').string)
            dict_page_good_boo.append(topic_content.find('div', class_='nrec').string)
            
        except AttributeError: #若是有人刪除文章，topic找不到會跑出這個錯誤
            continue
    
    if i%10 == 0:
        print('finished page:', i)

print('Finished PTT scarping!')
df = pd.DataFrame({
        'topic': dict_page_topic,
        'topic_URL': dict_page_topic_URL,
        'author': dict_page_author,
        'date': dict_page_date,
        'good_boo': dict_page_good_boo,
    })

df.tail()
df.to_csv('PTT_stock_p3000_p3718.csv')
df = pd.read_csv('PTT_stock_p3000_p3718.csv', index_col=0)
df<jupyter_output><empty_output><jupyter_text>近n天盤中閒聊推文內容<jupyter_code>n = 60 #最近n天的盤中推文
df[df['topic'].str.contains('盤中閒聊')][-n:]
daychat_URL = df[df['topic'].str.contains('盤中閒聊')]['topic_URL'][-n:].values
daychat_URL
driver.get(daychat_URL[0])
daychat = driver.page_source
soup = BeautifulSoup(daychat, 'lxml')
soup.find_all('div', class_='push')[0]
soup.find_all('div', class_='push')[0].find('span', class_='f3 push-content').string
driver.quit()
import time

push_type = []
push_ID = []
push_content = []
push_time = []
count = 0
start = time.time()

driver = webdriver.PhantomJS(executable_path='/Users/mac/Desktop/Programming/phantomjs-2.1.1-macosx/bin/phantomjs')

for url in daychat_URL:
    driver.get(url)
    daychat = driver.page_source
    soup = BeautifulSoup(daychat,'lxml')
    
    for topic_content in soup.find_all('div', class_='push'):
        
        try: #推文
            push_type.append(topic_content.find('span', class_='hl push-tag').string)
            push_ID.append(topic_content.find('span', class_='f3 hl push-userid').string)
            push_content.append(topic_content.find('span', class_='f3 push-content').string)
            push_time.append(topic_content.find('span', class_='push-ipdatetime').string)
        
        except AttributeError: #切換到噓文，用identation就可以一層一層處理異常了！！
            
            try:
                push_type.append(topic_content.find('span', class_='f1 hl push-tag').string)
                push_ID.append(topic_content.find('span', class_='f3 hl push-userid').string)
                push_content.append(topic_content.find('span', class_='f3 push-content').string)
                push_time.append(topic_content.find('span', class_='push-ipdatetime').string)
            
            except AttributeError: #連噓文都沒辦法，就給大宗師Exception忽略異常吧 
                print('連噓文都沒辦法，就忽略異常吧 <div class="push center warning-box">檔案過大！部分文章無法顯示</div>')
                continue
    
    count += 1
    
    if count % 10 == 0:
        print('finished chats:', count, 'time used(sec):', time.time()-start)

print('finished chat push scraping')
driver.quit()
df_daychat_push = pd.DataFrame({
        'type': push_type,
        'ID': push_ID,
        'content': push_content,
        'time': push_time
    })

df_daychat_push
df_daychat_push.to_csv('daychat_push_60d_1006.csv')
df_daychat_push = pd.read_csv('daychat_push_60d_1006.csv', index_col=0)
df_daychat_push['ID'].value_counts()
df_daychat_push['type'].value_counts()
len(set(df_daychat_push['ID'].values))
df_daychat_push.dropna(inplace=True) #有Nan會有method不好用
df_daychat_push.to_csv('daychat_push_60d_1006.csv')
df_daychat_push.shape
df_daychat_push[df_daychat_push['content'].str.contains('崩')]
df_daychat_push[df_daychat_push['content'].str.contains('多')]
df_daychat_push[df_daychat_push['content'].str.contains('可成')]<jupyter_output><empty_output><jupyter_text>對照用^TWII大盤指數: 2015/10~2017/10<jupyter_code>from pandas_datareader import data as web
TWII = web.DataReader(name='^TWII', data_source='yahoo', start='2015-10-01')
TWII.to_csv('TWII_20151001_20171006.csv')
TWII['up_1_down_0'] = np.where(TWII['Close']-TWII['Close'].shift(1)>0 , 1 , 0)
TWII['Pct_change'] = TWII['Close'].pct_change()*100 #percentage
TWII['Volatility level'] = np.where(np.abs(TWII['Pct_change'])>0.8 , 'high' , 'low') #用變動8%當界定線
TWII
TWII.to_csv('TWII_20151001_20171006.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16509498428317618, 'fraction_numerical': 0.02186688533552002, 'mean_word_length': 4.863782051282051, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '376042', 'license': 'MIT', 'n_tokens_mistral': 3108, 'n_tokens_neox': 3035, 'n_words': 304, 'path': 'bs4_PTT_stock .ipynb', 'repo_name': 'Pytoddler/Web-scraping', 'size': '7317'}