<jupyter_start><jupyter_text>wikipedia 한국어<jupyter_code>from urllib.request import urlopen
from bs4 import BeautifulSoup
import re
import csv
from urllib import parse

def loadPage(baseUrl, path, targetDir, depth, indexWriter, linkset):
    if path in linkset :
        return
    
    linkset.add(path)
    
    url = baseUrl + path
    print("loading ", url)
    
    try :
        html = urlopen(url)
        bsObj = BeautifulSoup(html, "html.parser")
        title = bsObj.find("h1", {"id":"firstHeading"}).text

        contentsDiv = bsObj.find("div", {"id":"mw-content-text"})
        contentsDiv = contentsDiv.find("div", {"class":"mw-parser-output"})
        
        paras = contentsDiv.findAll({"h1","h2","h3","h4","h5","h6","p"})
        
        links = []
        linkTitles = []
        for link in bsObj.find("div", {"id":"bodyContent"}).findAll("a", 
            href=re.compile("^(/wiki/)((?!:).)*$")):
              if 'href' in link.attrs:
                  links.append(link.attrs['href'])
                  linkTitles.append(link.text)

        filename = parse.quote(title)
        filename = filename.replace('/','_') + ".txt"
        filepath = targetDir + "/" + filename

        indexWriter.writerow([title, filename])
        
        with open(filepath, 'w', encoding='utf-8') as file :
            file.write(title)
            file.write("\n\n")
            
            
            for l in linkTitles:
                file.write(l)
                file.write("\n")
                
            for p in paras:
                if p.name == 'p':
                    file.write(p.text)
                else:
                    p = p.find("span",{"class":"mw-headline"})
                    if p:
                        file.write(p.text)
                file.write("\n")
            
                
            file.close()

        if(depth > 0):
            depth -= 1
            for link in links:
                loadPage(baseUrl, link, targetDir, depth, indexWriter, linkset)
    except Exception as ex :  
        print(ex)
    
        

# 경제사상사 : /wiki/%EA%B2%BD%EC%A0%9C%EC%82%AC%EC%83%81%EC%82%AC
economicThoughts = "/wiki/%EA%B2%BD%EC%A0%9C%EC%82%AC%EC%83%81%EC%82%AC"
# 경제학 : /wiki/%EA%B2%BD%EC%A0%9C%ED%95%99
economics = "/wiki/%EA%B2%BD%EC%A0%9C%ED%95%99"
socialogy = "/wiki/%EC%82%AC%ED%9A%8C%ED%95%99"
# 은닉마르코프 모델 : /wiki/%EC%9D%80%EB%8B%89_%EB%A7%88%EB%A5%B4%EC%BD%94%ED%94%84_%EB%AA%A8%EB%8D%B8
markov = "/wiki/%EC%9D%80%EB%8B%89_%EB%A7%88%EB%A5%B4%EC%BD%94%ED%94%84_%EB%AA%A8%EB%8D%B8"
# 체계이론 : /wiki/%EB%B6%84%EB%A5%98:%EC%B2%B4%EA%B3%84%EC%9D%B4%EB%A1%A0
system = "/wiki/%EB%B6%84%EB%A5%98:%EC%B2%B4%EA%B3%84%EC%9D%B4%EB%A1%A0"
# 보험
insurance = "/wiki/%EB%B3%B4%ED%97%98"

data_dir = "../../data/wiki2"

with open("../../data/wiki_page_index.csv", 'w', encoding='utf-8') as file :
    indexWriter = csv.writer(file)
    linkset = set()
    loadPage("https://ko.wikipedia.org", economics, 
             data_dir, 1, indexWriter, linkset)
    file.close()
import os
import sys
sys.path.insert(0, '../')
import pycor

model_size = 7000
outputpath = "../../output/" + str(model_size) + "/"
model_path = outputpath + "model/"

def listfiles(path):
    result_arr = []
    filenames = os.listdir(path)
    for filename in filenames:
        full_filename = os.path.join(path, filename)
        result_arr.append(full_filename)
    return result_arr
    
files = listfiles(data_dir)


print(pycor.version())
pycor.loadmodel(model_path)
_,words_array = pycor.readfile(files[1])

pycor.printSentences(words_array)

words_array, tags_array = pycor.__trim(words_array)

for words in words_array:
    print(words)

from pycor.res import WikiResolver

wikiResolver = WikiResolver()

pycor.addresolver(wikiResolver)

_,words_array = pycor.readfile(files[1])

keywords = wikiResolver.resolveKeywords(words_array, pycor.getmodel())

for word in keywords:
    head = word.bestpair.head
    print(head.text, head.pos)<jupyter_output>보험 {'C'}
공보험 {'C', 'SNG'}
사보험 {'C'}
상호보험 {'C', 'SNG'}
인보험 {'C'}
손해보험 {'C'}
상해보험 {'C', 'SNG'}
해상보험 {'C', 'SNG'}
항공보험 {'C', 'SNG'}
재보험 {'C'}
자동차보험 {'C'}
단체보험 {'C', 'SNG'}
기업보험 {'C', 'SNG'}
신용보험 {'C', 'SNG'}
보증보험 {'C'}
보험법 {'C', 'SNG'}
보험위부 {'C', 'SNG'}
보험약관 {'C', 'SNG'}
보험가액 {'C'}
보험사고 {'C'}
보험자 {'C'}
피보험자 {'C'}
보험대리상 {'C', 'SNG'}
보험중개인 {'C', 'SNG'}
보험모집인 {'C'}
보험의 {'C'}
보험수익자 {'C'}
보험보조자 {'C'}
보험계약 {'C'}
보험료 {'C'}
보험금액 {'C'}
실효약관 {'C', 'SNG'}
대표자책임이론 {'C'}
보험증권 {'C'}
손해보험 {'C'}
피보험이익 {'C'}
일부보험 {'C', 'SNG'}
초과보험 {'C', 'SNG'}
중복보험 {'C', 'SNG'}
보험자대위 {'C', 'SNG'}
잔존물대위 {'C'}
청구권대위 {'C', 'SNG'}
인보험 {'C'}
상법총칙 {'C'}
회사법 {'C', 'SNG'}
어음수표법 {'C', 'SNG'}
해상법 {'C'}
{'fraction_non_alphanumeric': 0.19611650485436893, 'fraction_numerical': 0.030420711974110032, 'mean_word_length': 2.705835331734612, 'pattern_counts': {'":': 5, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '603854', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2110, 'n_tokens_neox': 2258, 'n_words': 345, 'path': 'notebooks/wikipedia.ipynb', 'repo_name': 'ontolog-so/pycor', 'size': '4635'}