<jupyter_start><jupyter_text>1. selenium<jupyter_code>from bs4 import BeautifulSoup as bs
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
import time

# 페이지 열기
driver = webdriver.Chrome('./chromedriver.exe')
driver.get('https://cafe.daum.net/astronomical')
time.sleep(0.5)

# 로그인 클릭
driver.switch_to.frame('down')
elem = driver.find_element_by_xpath('//*[@id="loginout"]')
elem.click()
time.sleep(0.5)

# 카카오계정 로그인
elem = driver.find_element_by_xpath('//*[@id="mArticle"]/div[1]/div/div/div[2]/a[1]/span[2]')
elem.click()
time.sleep(0.5)

# 아이디 및 비밀번호 입력
with open('C:/Users/dojun/Documents/api_keys\daum.txt') as f:
    id = f.readline().replace('\n', '')
    pw = f.readline()

box_id = driver.find_element_by_xpath('//*[@id="id_email_2"]')
box_id.send_keys(id)

box_pw = driver.find_element_by_xpath('//*[@id="id_password_3"]')
box_pw.send_keys(pw)
box_pw.send_keys(Keys.ENTER)
time.sleep(0.5)

# 최신글 보기
driver.switch_to.frame('down')
elem = driver.find_element_by_xpath('//*[@id="fldlink_recent_bbs"]')
elem.click()
time.sleep(0.5)

# html 수집
html = driver.page_source
soup = bs(html, 'html.parser')
div = soup.find_all('span', class_='link_board')
del div[0:4]

# 링크 수집
ls_url = []
for d in div:
    href = d.find('a')['href']
    ls_url.append(href)
# 페이지 이동하며 html 수집
html = driver.page_source
soup = bs(html, 'html.parser')
div = soup.find_all('span', class_='link_board')
del div[0:4]

# 링크 수집
for d in div:
    href = d.find('a')['href']
    ls_url.append(href)

len(ls_url)<jupyter_output><empty_output><jupyter_text>*ls_url to data frame<jupyter_code>import pandas as pd

df = pd.DataFrame(ls_url)
df.to_csv('astronomy_url_list.csv')<jupyter_output><empty_output><jupyter_text>2. 페이지 이동하며 텍스트 수집<jupyter_code># 페이지 열기
driver = webdriver.Chrome('./chromedriver.exe')
driver.get('https://cafe.daum.net/astronomical')
time.sleep(0.5)

# 로그인 클릭
driver.switch_to.frame('down')
elem = driver.find_element_by_xpath('//*[@id="loginout"]')
elem.click()
time.sleep(0.5)

# 카카오계정 로그인
elem = driver.find_element_by_xpath('//*[@id="mArticle"]/div[1]/div/div/div[2]/a[1]/span[2]')
elem.click()
time.sleep(0.5)

# 아이디 및 비밀번호 입력
with open('C:/Users/dojun/Documents/api_keys\daum.txt') as f:
    id = f.readline().replace('\n', '')
    pw = f.readline()

box_id = driver.find_element_by_xpath('//*[@id="id_email_2"]')
box_id.send_keys(id)

box_pw = driver.find_element_by_xpath('//*[@id="id_password_3"]')
box_pw.send_keys(pw)
box_pw.send_keys(Keys.ENTER)
time.sleep(0.5)

# 페이지 이동
from tqdm import tqdm
base = 'https://cafe.daum.net/'

txt = ''
num = 0
for url in tqdm(ls_url, 'crawling...'):
    driver.get(base + url)
    driver.switch_to.frame('down')
    content = driver.find_element_by_id('user_contents').text
    txt += ' ' + content
    time.sleep(0.5)
len(txt)
with open('./astronomy.txt', 'w', encoding='utf-8') as f:
    f.write(txt)<jupyter_output><empty_output><jupyter_text>3. 형태소 분석<jupyter_code>from ckonlpy.tag import Twitter, Postprocessor

twitter = Twitter()
twitter.add_dictionary(['', ''], 'Noun')

passtags = {'Noun'}
postprocessor = Postprocessor(twitter, passtags=passtags)
tpl = postprocessor.pos(txt)
import pandas as pd

df = pd.DataFrame(tpl)
df
txt = ''
for word in df[0]:
    txt += ' ' + word
with open('./astronomy_nouns.txt', 'w', encoding='utf-8') as f:
    f.write(txt)
with open('./astronomy_nouns.txt', 'r', encoding='utf-8') as f:
    txt = f.read()<jupyter_output><empty_output><jupyter_text>4. 빈도 계산<jupyter_code>from collections import Counter

ls_words = txt.split()
ls_count = Counter(ls_words)
most_common = ls_count.most_common()
import pandas as pd
df = pd.DataFrame(most_common)
df.to_csv('./word_count.csv', encoding='cp949')<jupyter_output><empty_output><jupyter_text>5. 워드클라우드<jupyter_code>len(txt)
len(txt) / 3
smp1 = txt[:20000]
smp2 = txt[20000:40000]
smp3 = txt[40000:60000]
smp4 = txt[60000:80000]
smp5 = txt[80000:100000]
smp6 = txt[100000:]
from wordcloud import WordCloud
import matplotlib.pyplot as plt
from PIL import Image
import numpy as np

stopwords = {'것', '이', '입니다', '보정 프로그램', '있는', '촬영 정보', '보정', '프로그램', '부탁', '드립', '수',
            '및', '경', '고', '어요', '화천 조경철', '정보', '촬영 대상', '같습니다', '제출', '있다', '촬영', '그', '진행',
            '있습니다', '대상', '찍어 보', '화천'}

mask = np.array(Image.open('./wordcloud/frame18.png'))

my_wc = WordCloud(font_path='비트로 코어 TTF.ttf', 
                  width = 1000,
                  height = 1000,
                  background_color="black", 
                  max_words=10000, 
                  max_font_size=80, 
                  stopwords=stopwords,
                  mask=mask)

fig = plt.figure(figsize=(100, 100))
plt.imshow(my_wc.generate_from_text(smp6))
plt.axis('off')
plt.show()
fig.savefig('./wordcloud/word18.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18103092783505154, 'fraction_numerical': 0.027835051546391754, 'mean_word_length': 4.51877133105802, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '20536', 'license': 'MIT', 'n_tokens_mistral': 2201, 'n_tokens_neox': 2260, 'n_words': 349, 'path': '.ipynb_checkpoints/20210831_wordcloud_for_astronomy-checkpoint.ipynb', 'repo_name': 'DojunPark/TIL', 'size': '4850'}