<jupyter_start><jupyter_text>Get Industry & Sector List Credit : https://www.dataquest.io/blog/web-scraping-tutorial-python/<jupyter_code>import requests
import numpy as np
import pandas as pd
from bs4 import BeautifulSoup
from datetime import datetime<jupyter_output><empty_output><jupyter_text>ณ วันที่พัฒนานี้รองรับ URL "https://www.settrade.com/C13_MarketSummary.jsp?detail=INDUSTRY" ได้<jupyter_code>def getIndustryAndSector(url):
    page = requests.get(url) # ดูดหน้าเว็บมา
    
    if page.status_code == 200 :
        # All HTML of Page
        soup = BeautifulSoup(page.content, "html5lib")
        # ซอยข้อมูลทั้ง page เป็นทีละส่วนๆ แล้วลงลึกไปเอาข้อมูลทีละจุดๆ
        html = list(soup.children)[1]
        head = list(html.children)[0]
        body = list(html.children)[2]
        table_body = list(body.find(class_="table-hover"))[3]
        
        # Industry and Sector
        IAS_rows = table_body.find_all('tr')
        IAS_name_list = np.array([ IAS_row.find(class_="link-stt").text for IAS_row in IAS_rows ])
        IAS_url_list = np.array([ IAS_row.find(class_="link-stt").get('href') for IAS_row in IAS_rows ])
        
        # Only Industry
        Ind_rows = table_body.find_all(style="background: #cccccc;")
        Ind_name_list = np.array([ Ind_row.find(class_="link-stt").text for Ind_row in Ind_rows ])
        
        # Label Sector by Industry
        Ind_index = -1
        labels = np.array([])
        for IAS_name in IAS_name_list:
            if IAS_name in Ind_name_list:
                Ind_index+=1
            Ind_name = Ind_name_list[Ind_index]
            labels = np.append(labels, Ind_name)

        # Convert to DataFrame         
        df = pd.DataFrame({'Industry': labels, 'Sector': IAS_name_list, 'URL': IAS_url_list})
        industry = df[df['Industry'] == df['Sector']]
        del industry['Sector']
        sector = df[df['Industry'] != df['Sector']]
        return [industry,sector]
    else:
        print("Can't get content from this URL!!!")
[industry_df,sector_df] = getIndustryAndSector("https://www.settrade.com/C13_MarketSummary.jsp?detail=INDUSTRY")
industry_df
sector_df[0:10]
industry_df.to_csv('../../data/explore/industry_list.csv', index=False)
sector_df.to_csv('../../data/explore/sector_list.csv', index=False)
len(industry_df)
len(sector_df)
str(datetime.now().strftime('%Y-%m-%d'))<jupyter_output><empty_output><jupyter_text>Get Stock List<jupyter_code>sector_table = pd.read_csv('../../data/explore/sector_list.csv')
sector_table.head()<jupyter_output><empty_output><jupyter_text>สร้าง function Get หุ้นจาก แต่ละ Sector<jupyter_code>def getStockList(url,sector):
    page = requests.get(url) # ดูดหน้าเว็บมา
    
    if page.status_code == 200 :
        # All HTML of Page
        soup = BeautifulSoup(page.content, "html5lib")
        # ซอยข้อมูลทั้ง page เป็นทีละส่วนๆ แล้วลงลึกไปเอาข้อมูลทีละจุดๆ
        html = list(soup.children)[1]
        head = list(html.children)[0]
        body = list(html.children)[2]
        # ***
        target_table = list(body.find_all(class_="table-hover"))[1]
        target_table_body = list(target_table.children)[3]

        # Stocks
        stock_rows = target_table_body.find_all('tr')
        stock_name_list = np.array([ stock_row.find(class_="link-stt").text for stock_row in stock_rows ])
        stock_url_list = np.array([ stock_row.find(class_="link-stt").get('href') for stock_row in stock_rows ])

        # Convert to DataFrame         
        stock_df = pd.DataFrame({'Stock': stock_name_list, 'ราคาล่าสุด(Home)': stock_url_list})
        stock_df['Sector'] = sector
        return stock_df
    else:
        print("Can't get content from this URL!!!")
getStockList('https://www.settrade.com/C13_MarketSummary.jsp?detail=INDUSTRY&sector=FOOD&market=SET','FOOD').head()<jupyter_output><empty_output><jupyter_text>สร้าง Loop มาไล่วนดึง List หุ้นจากแต่ละ sector<jupyter_code>prefix_url = 'https://www.settrade.com'
stock_frames = []

for index,sector in sector_table.iterrows():
    suffix_url = sector['URL']
    settrade_url = prefix_url + suffix_url
    stock_df = getStockList(settrade_url,sector['Sector'])
    stock_frames.append(stock_df)

stock_lists = pd.concat(stock_frames)
stock_lists.head()
stock_lists.to_csv('../../data/explore/stock_list.csv', index=False)<jupyter_output><empty_output><jupyter_text>*Note : ตรงส่วนนี้ข้อมูลไม่ตรง จาก settrade ดึงหุ้นมาได้ 597 ตัว แต่ของกวีมีหุ้น 629 ตัว และข้อมูลเก่าที่เคยหาไว้มี 646 ตัว* ??<jupyter_code>len(stock_lists)<jupyter_output><empty_output><jupyter_text>Get URL of Stock<jupyter_code>stock_table = pd.read_csv('../../data/explore/stock_list.csv')
print(len(stock_table))
stock_table.head()<jupyter_output>597<jupyter_text>ข้อมูลจากตารางข้างบน URL จะชี้ไปหน้า **ราคาล่าสุด** แต่เราอยากไปหน้า **งบการเงิน** ดังนั้นต้องสร้างฟังก์ชั่น GET Financial Report URL ของหุ้น ก่อน<jupyter_code>def getStockLink(url,stock):
    page = requests.get(url) # ดูดหน้าเว็บมา
    
    if page.status_code == 200 :
        # All HTML of Page
        soup = BeautifulSoup(page.content, "html5lib")
        # ซอยข้อมูลทั้ง page เป็นทีละส่วนๆ แล้วลงลึกไปเอาข้อมูลทีละจุดๆ
        html = list(soup.children)[1]
        head = list(html.children)[0]
        body = list(html.children)[2]
        
        # ***
        nav_menu = list(body.find(class_="nav-tabs-stt"))
        
        previous_price_name = nav_menu[3].find('a').text #ราคาย้อนหลัง
        previous_price_url = nav_menu[3].find('a').get('href')
        company_stock_name = nav_menu[5].find('a').text #บริษัท/หลักทรัพย์
        company_stock_url = nav_menu[5].find('a').get('href') 
        board_of_director_name = nav_menu[7].find('a').text #กรรมการบริษัท
        board_of_director_url = nav_menu[7].find('a').get('href') 
        major_share_holder_name = nav_menu[9].find('a').text #ผู้ถือหุ้นรายใหญ่
        major_share_holder_url = nav_menu[9].find('a').get('href')  
        financial_name = nav_menu[11].find('a').text #งบการเงิน
        financial_url = nav_menu[11].find('a').get('href')

        # Convert to DataFrame         
        stock_df = pd.DataFrame({previous_price_name: [previous_price_url], 
                                 company_stock_name: [company_stock_url],
                                 board_of_director_name: [board_of_director_url],
                                 major_share_holder_name: [major_share_holder_url],
                                 financial_name: [financial_url]
                                })
        stock_df['Stock'] = stock
        return stock_df
    else:
        print("Can't get content from this URL!!!")
other_url = getStockLink('https://www.settrade.com//C13_FastQuote_Main.jsp?txtSymbol=CHOTI','CHOTI')
other_url<jupyter_output><empty_output><jupyter_text>สร้าง Loop ดึงข้อมูลหุ้นทุกตัว<jupyter_code>prefix_url = 'https://www.settrade.com'
stock_url_frames = []

for stock_index,stock in stock_table.iterrows():
    suffix_url = stock['ราคาล่าสุด(Home)']
    settrade_url = prefix_url + suffix_url
    stock_url_df = getStockLink(settrade_url,stock['Stock'])
    stock_url_frames.append(stock_url_df)

stock_url_lists = pd.concat(stock_url_frames)
stock_url_lists.head()<jupyter_output><empty_output><jupyter_text>merge เข้ากับตารางเดิม<jupyter_code>stock_url_table = pd.merge(stock_table,stock_url_lists, on=['Stock'], how='left')
print(len(stock_url_table))
stock_url_table.head()
stock_url_table.to_csv('../../data/explore/stock_url_list.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14563883702320618, 'fraction_numerical': 0.008268871699119766, 'mean_word_length': 3.7074701820464533, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 7, 'lorem ipsum': 0, 'www.': 7, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '204714', 'license': 'Apache-2.0', 'n_tokens_mistral': 3064, 'n_tokens_neox': 3041, 'n_words': 379, 'path': 'notebook/2_get_data/2_industry_sector_stock_list.ipynb', 'repo_name': 'Kanbc/stock-analysis', 'size': '7498'}