<jupyter_start><jupyter_text>Import Package<jupyter_code>import csv
from socket import MSG_MCAST
from urllib.parse import quote, quote_plus
import requests
from bs4 import BeautifulSoup
import pandas as pd
import re<jupyter_output><empty_output><jupyter_text>날짜 지정<jupyter_code>strDate = '20200901' # 시작 날짜
endDate = '20200930' # 종료 날짜<jupyter_output><empty_output><jupyter_text>CSV 저장<jupyter_code>filename = f"daum_news_{strDate}-{endDate}.csv"
f = open(filename, "w", encoding="utf-8-sig", newline="")
writer = csv.writer(f)<jupyter_output><empty_output><jupyter_text>유저 에이전트 지정
> [WhatismyUserAgent(ctrl+🖱 후 확인)](https://www.whatismybrowser.com/detect/what-is-my-user-agent)<jupyter_code>headers = {"User-Agent" : "개인 유저 에이전트"}<jupyter_output><empty_output><jupyter_text>반복문에 들어갈 날짜 지정<jupyter_code>dt_index = pd.date_range(start=strDate, end=endDate)
dt_list = dt_index.strftime("%Y%m%d").tolist()<jupyter_output><empty_output><jupyter_text>스크래핑 시작<jupyter_code>for i in dt_list:
    print('날짜',i)
    page_url = f"https://news.daum.net/breakingnews/?page=10000&regDate={i}"
    page =requests.get(page_url, headers=headers)
    page_soup = BeautifulSoup(page.text, "lxml")
    last_page = page_soup.find("em", attrs="num_page").get_text()
    lastPage_num = re.sub(r'[^0-9]','',last_page)
    # print(lastPage_num)

    for j in range(1, int(lastPage_num)+1):
        main_url = f"https://news.daum.net/breakingnews/?page={j}&regDate={i}" # url 입력
        res = requests.get(main_url, headers=headers)
 
        if res.status_code == 200:
            print(i, int(lastPage_num), '중' ,j,'page',round(j/int(lastPage_num)*100, 2),'%', main_url, 'status:',res.status_code)
            soup = BeautifulSoup(res.text, "lxml") # soup으로 저장
            main = soup.find("ul", attrs={"class":"list_news2 list_allnews"})
            news = main.find_all("strong", attrs={"class":"tit_thumb"})
            cnt = 0
            for new in news:
                urls = new.select_one("a")["href"]# 페이지에 나와있는 뉴스 URL 변수 입력
                # print(urls)
                result = requests.get(urls, headers=headers)         # request 로 다시 개별 뉴스 접속
                if result.status_code == 200:
                    news_soup = BeautifulSoup(result.text, "lxml")
                    # 뉴스 제목, 발행시간, 기사본문 저장
                    title = news_soup.find("h3", attrs={"tit_view"}).get_text().strip()
                    pubdate = news_soup.find("span", attrs={"num_date"}).get_text().strip()
                    text = news_soup.find("div", attrs={"news_view"}).get_text().strip()
                    cnt += 1
                    # print(j,'of',cnt,'번째 기사')
                    # print(i,j,'of',cnt,'번째 기사', urls,'status:', result.status_code)
                    writer.writerow([cnt, title, pubdate, urls, text])
                else:
                    print(i,j,'of',cnt,'번째 기사','error_code :',result.status_code, urls)
                    pass
                   
        else:
            print(i,'page : ',j,'error_code :',res.status_code, main_url)
            pass<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1500638569604087, 'fraction_numerical': 0.013090676883780333, 'mean_word_length': 2.8160779537149816, 'pattern_counts': {'":': 2, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '512965', 'license': 'Apache-2.0', 'n_tokens_mistral': 1238, 'n_tokens_neox': 1273, 'n_words': 179, 'path': 'scraping/daumNews_scrap_baseline.ipynb', 'repo_name': 'young-ha713/HEXinAR_exawave_service', 'size': '3132'}