<jupyter_start><jupyter_text>Imports<jupyter_code># you need to install chromedriver

import selenium
from selenium import webdriver
import bs4
import lxml
import pandas as pd
import time<jupyter_output><empty_output><jupyter_text>Init DataFrame<jupyter_code># init dataframe
# sentiment gonna be caculated by specific method has standard
df = pd.DataFrame(columns=['id','title','review','score','sentiment'])
df<jupyter_output><empty_output><jupyter_text>Web Crawling<jupyter_code># get naver movie reviews
base_url = 'https://movie.naver.com/movie/point/af/list.naver'

# execute chrome web driver
driver = webdriver.Chrome('./chromedriver.exe')
for p in range(1,1001):
    driver.get(''.join([base_url,'?&page=',str(p)])) # get target page
    driver.implicitly_wait(time_to_wait=3) # in second
    
    html = bs4.BeautifulSoup(driver.page_source,'lxml')
    reviews:bs4.element.ResultSet = html.find('table',{'class':'list_netizen'}).find('tbody').find_all('tr')
    
    lst = []
    # get id, title, review text and score
    for r in reviews:
        ac_num = r.find('td',{'class':'ac num'}).text
        t = r.find('td',{'class':'title'})
        title = t.find('a').text
        score = float(t.find('em').text)
        
        t.a.decompose()
        t.span.decompose()
        t.div.decompose()
        t.br.decompose()
        t.a.decompose()
        
        text = t.get_text(strip=True)
        
        lst.append([ac_num,title,text,score,None])
    df = pd.concat([df,pd.DataFrame(lst,columns=['id','title','review','score','sentiment'])],ignore_index=True)

driver.close()

df.to_csv()
df<jupyter_output><empty_output><jupyter_text>Save File<jupyter_code>df.to_csv(''.join(['./data/reviews_',str(int(time.time())),'.csv']),index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16638655462184873, 'fraction_numerical': 0.005042016806722689, 'mean_word_length': 4.363363363363363, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '544338', 'license': 'MIT', 'n_tokens_mistral': 657, 'n_tokens_neox': 635, 'n_words': 101, 'path': '1_kdh_review_crawler.ipynb', 'repo_name': 'tmxkqotnl/project_mini_nlp', 'size': '1785'}