<jupyter_start><jupyter_text>web crawling- 1. requests : json : 동적페이지(URL 변경 없이 데이터를 수정 및 추가)- 2. requests : html : 정적페이지(URL 변경으로 데이터를 수정 및 추가)- 3. selenium : web browser : 1, 2번 방법을 사용하지 못할때 사용  1. 네이버 주식 데이터 크롤링- 코스피 데이터 수집- 코스닥 데이터 수집- USD환율 데이터 수집- 그래프 그리기- 상관계수를 확인  크롤링 절차- 1. 웹서비스 분석 : URL 분석하기- 2. request, response 하기 : json 얻기- 3. 문자열 json 데이터를 dict로 변경(파싱) 후 데이터프레임으로 생성<jupyter_code>import requests

# 1. 웹 서비스 분석 : URL 찾기

code, page_size, page = "KOSPI",20,1

url = "https://m.stock.naver.com/api/json/sise/dailySiseIndexListJson.nhn?\
code={}&pageSize={}&page={}".format(code, page_size,page)

url
# 2. request, response : JSON(str) -> JSON(dict) -> DF
response = requests.get(url)
response
#JSON(str) -> JSON(dict) -> DF
datas = response.json()["result"]["siseList"]
type(datas), len(datas)
kospi_df = pd.DataFrame(datas)
kospi_df.tail(3)<jupyter_output><empty_output><jupyter_text>KOSDAQ 크롤링<jupyter_code>def get_stock_data(code,page_size=20,page=1):
    url="https://m.stock.naver.com/api/json/sise/dailySiseIndexListJson.nhn?\
code={}&pageSize={}&page={}".format(code,page_size,page)
    response = requests.get(url)
    datas=response.json()['result']['siseList']
    return pd.DataFrame(datas)<jupyter_output><empty_output><jupyter_text>USD 크롤링<jupyter_code>def get_currency_data(code,page_size=100,page=1):
    url = 'https://m.stock.naver.com/api/json/marketindex/marketIndexDay.nhn?\
marketIndexCd={}&pageSize={}&page={}'.format(code,page_size,page)
    params={
    "cookie":"NNB=IXOI6NVFRWIF4; NRTK=ag#all_gr#1_ma#-2_si#0_en#0_sp#0; _ga=GA1.2.310078303.1590834359; ASID=6af24fe600000172ea150b9f00000057; page_uid=UIZkMdp0Jy0ssdBgU6Gssssstgd-332041; my_last_menu_id=-2; nid_inf=921058332; NID_AUT=7cKi8IMtYELnPyQ15zYEYMNyE8bUmzrkvrtoKHkx5gqCfl0PpXQBwL07wHpgYkdC; NID_JKL=aywq7xpPPb2Gm/IaHyBDekKmLsxaQ0J42lqip1ydAyk=; NID_SES=AAABch/cACsXr23ZjNUVpSYuJIcNJ+UzNf3FC6ji8H6MT52xkoDXdSX91xt9Wm9LUmZIBaE20svfIrYIgaAJ+Ik3bhxpuFDKGCxhxdrZ8JtVTubW7g6tfT8MYbBKugQCQY21Wuy4v3iEaOUHDwad/VaHIkPz1gMpXaVpbCZXRlFiU3UMQTdxjLKeShfiV+sncZwzfpiciYRax+nEq0uQ9Ovnwn5So/7ysiVWtH6JQmtUzHN1WnMVrDu8JRip0wKyMWtGaDJ4nghsV5yDhRQ4YRwIA1ZpREsi04O4tfech7vNSl34gA7rJ4ssFgsFrRLnJqFLDmuavCg3MS9XpIFnp5fRg2e3TfO/jqngid7rOHoFRxQSBwDG3/NzlRkZMLLEkH1wna3E8rsHN5xK7mgsvfdFw+PK6My5Bn2J+imVEvZ6vDky1LXbz8OJJRuThvXQ2eQ8m0rkK0D79JpmBYkV9aQ9naADWutG7jjWzblRN1yigwxZ7+I84jt7hQLx0Vejmd7jJw==; JSESSIONID=4E5F94135E1CEE4F4D935EE9DC35F3C9; BMR=; XSRF-TOKEN=<PASSWORD>"}
    response = requests.get(url,headers=params)
    
    data=response.json()['result']['marketIndexDay']
    return pd.DataFrame(data)<jupyter_output><empty_output><jupyter_text>그래프로 그리기<jupyter_code>%matplotlib inline
%config InlineBackend.figure_formats = {'png', 'retina'}
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns
sns.set()
kosdaq_df=get_stock_data("KOSDAQ",100)
kospi_df=get_stock_data("KOSPI",100)
USD=get_currency_data("FX_USDKRW",100)
merge_df_1 = pd.merge(kosdaq_df,kospi_df,left_on="dt",right_on="dt")
merge_df_1.tail(2)
merge_df_1 = merge_df_1.rename(columns={"ncv_x":"ncv_kosdaq","ncv_y":"ncv_kospi"})
merge_df_1 = merge_df_1[["dt","ncv_kosdaq","ncv_kospi"]]
merge_df_1.tail(2)
merge_df = pd.merge(merge_df_1,USD)[["dt","ncv_kosdaq","ncv_kospi","nv"]]
plt.figure(figsize=(20,5))
plt.plot(merge_df["dt"],merge_df["ncv_kospi"],"*r:",label="kospi")
plt.plot(merge_df["dt"],merge_df["ncv_kosdaq"],"bo",label="kosdaq")
plt.plot(merge_df["dt"],merge_df["nv"],"g*",label="USD")

plt.xticks(merge_df_1[::len(merge_df)//10]["dt"])<jupyter_output><empty_output><jupyter_text>정규화 : Nomalization- `z = (x - min(x)) / (max(x) - min(x))`<jupyter_code>from sklearn import preprocessing
preprocessing.minmax_scale(kospi_df[::-1]["ncv"])
# 코스피, 코스닥 종가 데이터 그리기
plt.figure(figsize=(20,5))
plt.plot(merge_df["dt"],merge_df["ncv_kospi"],"*r:",label="kospi")
plt.plot(merge_df["dt"],merge_df["ncv_kosdaq"],"bo",label="kosdaq")
plt.plot(merge_df["dt"],merge_df["nv"],"g*",label="USD")
plt.xticks(merge_df_1[::len(merge_df_1)//10]["dt"])
# 코스피, 코스닥, 환율 종가 데이터 그리기
plt.figure(figsize=(20, 5))

plt.plot(kospi_df[::-1]["dt"], 
         preprocessing.minmax_scale(kospi_df[::-1]["ncv"]), 
         "r*:", label="kospi")
plt.plot(kosdaq_df[::-1]["dt"], 
         preprocessing.minmax_scale(kosdaq_df[::-1]["ncv"]), 
         "bo:", label="kosdaq")
plt.plot(USD[::-1]["dt"], 
         preprocessing.minmax_scale(USD[::-1]["nv"]), 
         "g^--", label="usd")
plt.legend(loc=0)

plt.show()
import numpy as np
np.corrcoef(kospi_df["ncv"], kosdaq_df["ncv"])[0,1]
np.corrcoef(kospi_df["ncv"], USD["nv"])[0, 1]
np.corrcoef(kosdaq_df["ncv"], USD["nv"])[0, 1]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1745110681280894, 'fraction_numerical': 0.054588437567160975, 'mean_word_length': 8.107632093933464, 'pattern_counts': {'":': 3, '<': 22, '<?xml version=': 0, '>': 26, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '464512', 'license': 'MIT', 'n_tokens_mistral': 2591, 'n_tokens_neox': 2567, 'n_words': 230, 'path': 'crawling/01_requests_1.ipynb', 'repo_name': 'yeonghwanchoi/TIL', 'size': '4679'}