<jupyter_start><jupyter_text>TopicModeling : https://wikidocs.net/30707Word2Vec : https://wikidocs.net/22660<jupyter_code>!curl -O https://raw.githubusercontent.com/franciscadias/data/master/abcnews-date-text.csv

# 15년간 발행된 뉴스의 기사 제목 데이터
import pandas as pd
df_data = pd.read_csv('./abcnews-date-text.csv')
df_data.head(5)
df_data = df_data.head(10000)
head_text = df_data[['headline_text']]
type(head_text)<jupyter_output><empty_output><jupyter_text>Data preprocessing<jupyter_code>import nltk
nltk.download('punkt')
# apply : map() 같은 기능 

head_text['title_text'] = head_text.apply(lambda row : nltk.word_tokenize(row['headline_text']), axis=1)  
# 텍스트화 
head_text.head(3)
from nltk.corpus import stopwords
nltk.download('stopwords')
stop = stopwords.words('english')
print(stop)<jupyter_output>['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're", "you've", "you'll", "you'd", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', "she's", 'her', 'hers', 'herself', 'it', "it's", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', "that'll", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', '[...]<jupyter_text>- 단어 길이가 3 이하인 것도 불용어 처리하기```def stopWord(head_text):  result = []  for x in head_text['title_text']:    for word in x:      if word not in stop or len(word) > 3:        result.append(word)  return resultdef callStopWord(head_text):  titles = []  for x in head_text['title_text']:    titles.append(word)  return titles  ```<jupyter_code># 길이가 3 이하인 것도 불용어 처리하기
# 람다 버전

head_text['title'] = head_text['title_text'].apply(lambda x: [ word for word in x if (len(word) > 3) if (word not in stop)])
head_text.head(5)
head_text['title'][3]
tokens = []
for i in range(len(head_text)):
  tokens.append( ' '.join(head_text['title'][i]) )  # 리스트 요소를 문자화 시키기

tokens[3:5]
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf = TfidfVectorizer(max_features=1000)
X = tfidf.fit_transform(tokens)
X.shape
# X[4].toarray()
from sklearn.decomposition import LatentDirichletAllocation  
# 단어 중요도 나타내기
lda_model = LatentDirichletAllocation( n_components=4, max_iter=10 )  
# n_components : 현재 데이터에서 topic분류 개수,  max_itr : epochs와 같은 개념
lda_top = lda_model.fit_transform(X)
lda_model.components_.shape,  lda_model.components_  # numpy로 중요도가 담김
terms = tfidf.get_feature_names()  # 단어 모음
n = 5   # 5개씩 뽑음
for idx, topic in enumerate(lda_model.components_):
  print( [(terms[i], topic[i]) for i in topic.argsort()[:-n-1:-1]] )

# topic의 주요 단어 -> 부정적인 내용의 기사가 많군..<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21354979706525132, 'fraction_numerical': 0.012800499531689042, 'mean_word_length': 5.875536480686695, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 15, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '363609', 'license': 'Apache-2.0', 'n_tokens_mistral': 1413, 'n_tokens_neox': 1431, 'n_words': 309, 'path': 'topicmodeling_LDA.ipynb', 'repo_name': 'mmeooo/test_NLP', 'size': '3203'}