<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import urllib.request

urllib.request.urlretrieve("https://raw.githubusercontent.com/franciscadias/data/master/abcnews-date-text.csv", filename="abcnews-date-text.csv")
data = pd.read_csv('abcnews-date-text.csv', error_bad_lines=False)
print(len(data))
print(data.head(5))
text = data[['headline_text']]
text.head(5)
import nltk
nltk.download('punkt')
text['headline_text'] = text.apply(lambda row : nltk.word_tokenize(row['headline_text']), axis=1)
print(text.head(5))
from nltk.corpus import stopwords
nltk.download('stopwords')
stop = stopwords.words('english')
text['headline_text'] = text['headline_text'].apply(lambda x : [word for word in x if word not in (stop)])
from nltk.stem import WordNetLemmatizer
nltk.download('wordnet')
text['headline_text'] = text['headline_text'].apply(lambda x: [WordNetLemmatizer().lemmatize(word, pos='v') for word in x])
print(text.head(5))<jupyter_output>[nltk_data] Downloading package wordnet to /root/nltk_data...
[nltk_data]   Unzipping corpora/wordnet.zip.
                                       headline_text
0       [aba, decide, community, broadcast, licence]
1      [act, fire, witness, must, aware, defamation]
2      [g, call, infrastructure, protection, summit]
3          [air, nz, staff, aust, strike, pay, rise]
4  [air, nz, strike, affect, australian, travellers]<jupyter_text><jupyter_code>tokenized_doc = text['headline_text'].apply(lambda x: [word for word in x if len(word) >3])
print(tokenized_doc[:5])
# 역토큰화 (토큰화 작업을 되돌림)
detokenized_doc = []
for i in range(len(text)):
  t = ' '.join(tokenized_doc[i])
  detokenized_doc.append(t)

text['headline_text'] = detokenized_doc # 역 토큰화된 데이터를 다시 저장
text['headline_text'][:5]
from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(stop_words='english', max_features=1000) # 상위 1000개의 단어를 보존
X = vectorizer.fit_transform(text['headline_text'])
X.shape
from sklearn.decomposition import LatentDirichletAllocation
lda_model = LatentDirichletAllocation(n_components=10, learning_method='online', random_state=777, max_iter=1)
lda_top = lda_model.fit_transform(X)
print(lda_model.components_)
print(lda_model.components_.shape) 
terms = vectorizer.get_feature_names() # 단어 집합, 1000개의 단어가 저장됨.

def get_topics(components, feature_names, n=5):
  for idx, topic in enumerate(components):
    print("Topic %d:" % (idx+1), [(feature_names[i], topic[i].round(2)) for i in topic.argsort()[:-n - 1:-1]])
get_topics(lda_model.components_, terms)<jupyter_output>Topic 1: [('government', 8725.19), ('sydney', 8393.29), ('queensland', 7720.12), ('change', 5874.27), ('home', 5674.38)]
Topic 2: [('australia', 13691.08), ('australian', 11088.95), ('melbourne', 7528.43), ('world', 6707.7), ('south', 6677.03)]
Topic 3: [('death', 5935.06), ('interview', 5924.98), ('kill', 5851.6), ('jail', 4632.85), ('life', 4275.27)]
Topic 4: [('house', 6113.49), ('2016', 5488.19), ('state', 4923.41), ('brisbane', 4857.21), ('tasmania', 4610.97)]
Topic 5: [('court', 7542.74), ('attack', 6959.64), ('open', 5663.0), ('face', 5193.63), ('warn', 5115.01)]
Topic 6: [('market', 5545.86), ('rural', 5502.89), ('plan', 4828.71), ('indigenous', 4223.4), ('power', 3968.26)]
Topic 7: [('charge', 8428.8), ('election', 7561.63), ('adelaide', 6758.36), ('make', 5658.99), ('test', 5062.69)]
Topic 8: [('police', 12092.44), ('crash', 5281.14), ('drug', 4290.87), ('beat', 3257.58), ('rise', 2934.92)]
Topic 9: [('fund', 4693.03), ('labor', 4047.69), ('national', 4038.68), ('council', 40[...]
{'fraction_non_alphanumeric': 0.19353023909985936, 'fraction_numerical': 0.08635724331926864, 'mean_word_length': 5.972549019607843, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '376455', 'license': 'MIT', 'n_tokens_mistral': 1590, 'n_tokens_neox': 1412, 'n_words': 294, 'path': 'colab/Latant_Dirichlet_Allocatic.ipynb', 'repo_name': 'cateto/python4NLP', 'size': '3555'}