<jupyter_start><jupyter_text>![alt text](data.png 'Logoo Title Text 1')  HTML 데이터 파싱을 이용한 크롤링    - 네이버 연관 검색어        - 1. 웹페이지 분석 : URL 찾기        - 2. 요청 > 응답 : HTML(str)        - 3. HTML(str) > BeautifulSoup obh (css-selector) > dataframe            - 1. select : 여러개의 element 객체를 리스트로 가져옴            - 2. select_one : 하나의 element 객체를 가져옴<jupyter_code># 1.웹페이지 분석 : URL
query='삼성전자'
url = 'https://search.naver.com/search.naver?sm=top_hty&fbm=1&ie=utf8&query={}'.format(query)
# 2.resquests > response : HTML(str)
response = requests.get(url)
# print(url)
response
# 3.HTML(str) > BeautifulSoup obj (css-selector) > dataframe
resp = BeautifulSoup(response.content, 'html.parser')
elements = resp.select('ul.type01')[2]

news_title = []
for element in elements.select('li > dl > dt'):
#     print(element)
    news_title.append(element.text)
print(news_title)
for element in elements.select('li > dl > dt > a'):
    news_title = [element for element in elements]
print(news_title)
# 함수로 만들기
def naver_news_title(query):
    query='삼성전자'
    url = 'https://search.naver.com/search.naver?sm=top_hty&fbm=1&ie=utf8&query={}'.format(query)
    # 2.resquests > response : HTML(str)
    response = requests.get(url)
    # print(url)
    response
    # 3.HTML(str) > BeautifulSoup obj (css-selector) > dataframe
    resp = BeautifulSoup(response.content, 'html.parser')
    elements = resp.select('ul.type01')[2]
    news_title = []
    for element in elements.select('li > dl > dt'):
    #     print(element)
        news_title.append(element.text)
    return news_title
result = naver_news_title(query)
result
df = pd.DataFrame(result)
df<jupyter_output><empty_output><jupyter_text>다음 뉴스 데이터        - 다음 뉴스의 메인 페이지 15개기사의 제목, 링크, 내용 수집<jupyter_code># 1.URL
url = 'https://news.daum.net/'
# 2.HTML(str)
response = requests.get(url)
# 3.BS
bs = BeautifulSoup(response.content, 'html.parser')
elements = bs.select('.list_headline > li:not(.item_main)')
len(elements)
# 4.
elements
datas = []
for element in elements:
    datas.append({
        'title':element.select_one('.tit_g').text.strip().replace('\n', ''),
        'link' :element.select_one('a').get('href')
    })
articles_df = pd.DataFrame(datas)
articles_df
# content 데이터 추가
link = articles_df.loc[0, 'link']
print(link)
response = requests.get(link)
resp = BeautifulSoup(response.content, 'html.parser')
contents = resp.select_one('#harmonyContainer').text.strip().replace('\n', '')
contents
def get_content(link):
    response = requests.get(link)
    resp = BeautifulSoup(response.content, 'html.parser')
    contents = resp.select_one('#harmonyContainer').text.strip().replace('\n', '')
    return contents
articles_df['contents'] = articles_df['link'].apply(get_content)
articles_df.tail()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15711711711711712, 'fraction_numerical': 0.01009009009009009, 'mean_word_length': 4.102941176470588, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 26, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '611675', 'license': 'MIT', 'n_tokens_mistral': 1100, 'n_tokens_neox': 1144, 'n_words': 198, 'path': 'python/Crawling/08_HTML_daum.ipynb', 'repo_name': 'WGLee87/TIL', 'size': '2775'}