<jupyter_start><jupyter_text>네이버<jupyter_code>all_articles = []
data = defaultdict(list)
parse_time_per_page = []

for page_index in range(100):
    article_index = page_index * 10 + 1
    
    url = f"https://search.naver.com/search.naver?where=news&query={search_keyword}&sort=1&start={article_index}"
    
    start = time.time()
    raw = requests.get(url, headers={'User-Agent':'Mozilla/5.0'})
    
    if raw.status_code == 404:
        logger.warning("404 at {url}")
        continue
        
    html = BeautifulSoup(raw.text, "html.parser")

    articles = html.select("ul.type01 > li")
    
    for i, article in enumerate(articles):
        title = article.select_one("a._sp_each_title").text
        outlet = article.select_one("span._sp_each_source").text
        date = re.sub(outlet, '', re.sub('보내기', '', article.select_one("dd.txt_inline").text)).strip()
        
        if outlet.endswith(' 선정'):
            pick = True
        else:
            pick = False
        
        outlet = outlet.strip(' 선정')
        
        data['title'].append(title)
        data['outlet'].append(outlet)
        data['date'].append(date)
        data['pick'].append(pick)
        
    end = time.time()
    
    parse_time_per_page.append(end - start)
    articles = html.select("ul.type01 > li")
    select_ids = []
    
    for i, article in enumerate(articles):
        outlet = article.select_one("span._sp_each_source").text
        outlet = outlet.strip(" 선정")
        if outlet in outlets_of_interest:
            print(outlet, end='\r')
            select_ids.append(i)
            
    all_articles.extend([articles[j] for j in select_ids])
    
np.max(parse_time_per_page)
all_articles[0]
old_article = requests.get("https://news.naver.com/main/read.nhn?mode=LSD&amp;mid=sec&amp;sid1=101&amp;oid=020&amp;aid=0000208705")
old_article.status_code<jupyter_output><empty_output><jupyter_text>한겨레 기사<jupyter_code>len(all_articles)
all_articles
for a in articles:
    print(a.select_one("span._sp_each_source").text)
"아시아경제언론사"
articles[0]
articles[0].select_one("dd.txt_inline").text
len(data['title'])
df = pd.DataFrame(data)
df.head(10)
df.pick.sum()
df[df.outlet.apply(lambda x: x in outlets_of_interest)].shape
df.to_csv("../data/processed/naver_titles_5000.csv", index=False)
np.sum(parse_time_per_page)
783.1217935085297 / 60
b = requests.get("https://search.naver.com/search.naver?where=news&query=%22%EA%B8%B0%EB%B3%B8%EC%86%8C%EB%93%9D%22&sm=tab_opt&sort=1&photo=0&field=0&reporter_article=&pd=0&ds=&de=&docid=&nso=so%3Add%2Cp%3Aall%2Ca%3Aall&mynews=1&refresh_start=0&related=0")
html = BeautifulSoup(b.text, "html.parser")
articles = html.select("ul.type01 > li")
for aa in articles:
    print(aa.select_one("span._sp_each_source").text)<jupyter_output>업코리아
UPI뉴스
MTN
월간노동법률
연합뉴스
한국농정신문
이뉴스투데이
일간투데이
쿠키뉴스
노동과희망<jupyter_text>다음<jupyter_code>search_keyword = '"<PASSWORD>"'
cp_dict = {
    '조선일보': '16d4PV266g2j-N3GYq',
    '한겨레': '16nzyJHdH5ORpabfqG',
    '경향신문': '16akMkKFDu6n8GTzZr',
    '동아일보': '16bOiOx4gG2S18EPLj',
}

cp_key = cp_dict['한겨레']
page_num=1

daum_url = f"https://search.daum.net/search?w=news&q={search_keyword}&cp={cp_key}&DA=PGD&p={page_num}&cluster=n"
r = requests.get(daum_url)
html = BeautifulSoup(r.text, "html.parser")
articles = html.select("ul#newsResultUL > li")
for i, ar in enumerate(articles):
    article_url = ar.select_one('a.f_nb')['href']
    article_res = requests.get(article_url)
    article_html = BeautifulSoup(article_res.text, "html.parser")
    title = article_html.select_one("h3.tit_view").text
    date = article_html.select_one("span.num_date").text
    body = article_html.select_one("div.news_view").text
    break
title
date
body.strip()
paragraphs = article_html.select("div#harmonyContainer.article_view > section > p")
text = '\n\n'.join([p.text.strip() for p in paragraphs])
summary = article_html.select_one("strong.summary_view").text.strip()
summary
cnt_text = html.select_one("span#resultCntArea").text
cnt = cnt_text.split("/")[1]

int(''.join(re.findall('\d+', cnt)))
cnt_text
import os

for f in os.listdir("../data"):
    print(f.__dict__)
    break<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15965376292377975, 'fraction_numerical': 0.02765087761481125, 'mean_word_length': 4.19350811485643, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '604849', 'license': 'RSA-MD', 'n_tokens_mistral': 1738, 'n_tokens_neox': 1730, 'n_words': 215, 'path': 'notebooks/1.1-crawling.ipynb', 'repo_name': 'suzinyou/ubi_articles', 'size': '4153'}