<jupyter_start><jupyter_text><jupyter_code>import tweepy
import nltk
from tweepy import OAuthHandler
from nltk.tokenize import word_tokenize
from nltk.tokenize import TweetTokenizer
#from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
import os
#from google.colab import drive
#drive.mount('/content/drive/')
from google.colab import drive
drive.mount('/gdrive')

# create stemmer
#factory = StemmerFactory()
#stemmer = factory.create_stemmer()

tknzr = TweetTokenizer()
consumer_key = 'UOh0XJeH9kwDt8aWzEQCqPsc5'
consumer_secret = '<KEY>'
access_token = '<KEY>'
access_secret = '<KEY>'

auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)

api = tweepy.API(auth)
for status in tweepy.Cursor(api.home_timeline).items(100):

# Process a single status

  # print(status.text)
   case_folding =status.text.lower()
  # print(case_folding)
  # output   = stemmer.stem(case_folding)
   print(case_folding)
 
   #tokens=tknzr.tokenize(output)
   #print(tokens)
   
   file1 = open("/gdrive/My Drive/MyFile76.txt","a") 
   file1.write(case_folding)
   file1.close()
from google.colab import drive
drive.mount('/gdrive/My Drive')




with open('/gdrive/My Drive/foo.txt', 'w') as f:
  f.write('Hello Google Drive!')
!cat '/gdrive/data/MyFile3.txt'
!cat '/gdrive/My Drive/MyFile55.txt'
!ls -l '/gdrive/My Drive'
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from sklearn.metrics import adjusted_rand_score

f=open("/gdrive/My Drive/MyFile76.txt", "r")
contents = f.readlines()
#contents =f.read()

print(contents)
#contents=[contents.replace('\n','')]

print(contents)

vectorizer = TfidfVectorizer(stop_words='english')
X = vectorizer.fit_transform(contents)

true_k = 5
model = KMeans(n_clusters=true_k, init='k-means++', max_iter=50, n_init=1)
model.fit(X)

print("Top terms per cluster:")
order_centroids = model.cluster_centers_.argsort()[:, ::-1]
terms = vectorizer.get_feature_names()
for i in range(true_k):
    print("Cluster %d:" % i),
    for ind in order_centroids[i, :10]:
        print(' %s' % terms[ind]),
    print

print("\n")
f.close()

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from sklearn.metrics import adjusted_rand_score

import re
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from scipy.spatial.distance import pdist
from scipy.cluster.hierarchy import dendrogram, linkage, fcluster



documents = ["This little kitty came to play when I was eating at a restaurant.",
             "Merley has the best squooshy kitten belly.",
             "Google Translate app is incredible.",
             "If you open 100 tab in google you get a smiley face.",
             "Best cat photo I've ever taken.",
             "Climbing ninja cat.",
             "Impressed with google map feedback.",
             "Key promoter extension for Google Chrome."]

tfidf = TfidfVectorizer(stop_words='english')
X = tfidf.fit_transform(documents).todense()
# transform the data matrix into pairwise distances list
dist_array = pdist(X)
# calculate hierarchy
#Z = linkage(dist_array, 'ward')
#plt.title("Ward")
#dendrogram(Z, labels=labels)

import scipy.cluster.hierarchy as sch
dendrogram = sch.dendrogram(sch.linkage(X, method = 'ward'))
plt.title('Dendrogram')
plt.xlabel('Nomor Dokumen')
plt.ylabel('Jarak Euclidean')
plt.show()

import glob
path = "/gdrive/My Drive/"
list = []
for files in glob.glob(path +"*.txt"):
    print(files)
    infile = open(files)
    a = infile.readline().split('.')
    for k in range (0,len(a)):
        print(a[0])
        list.append(a[0])
infile.close()
print(list)
tfidf = TfidfVectorizer(stop_words='english')
X = tfidf.fit_transform(list).todense()
# transform the data matrix into pairwise distances list
dist_array = pdist(X)
# calculate hierarchy
#Z = linkage(dist_array, 'ward')
#plt.title("Ward")
#dendrogram(Z, labels=labels)

import scipy.cluster.hierarchy as sch
dendrogram = sch.dendrogram(sch.linkage(X, method = 'ward'))
plt.title('Dendrogram')
plt.xlabel('Nomor Dokumen')
plt.ylabel('Jarak Euclidean')
plt.show()
print("done")

from sklearn.cluster import AgglomerativeClustering

cluster = AgglomerativeClustering(n_clusters=2, affinity='euclidean', linkage='ward')  
cluster.fit_predict(X) 
print(cluster.labels_)  
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from sklearn.metrics import adjusted_rand_score

documents = ["jelang putus mk polisi di bogor gencar patroli tengah malam https t co 8mdr0z4fhe",
             "ada banyak alas mengapa total belanja orang indonesia untuk libur bilang tidak besar negara lain di asia https t co n0cpgkxpqx",
             "barang bukti upa uang rp 35 juta kembali karena vanessa tak bukti laku prostitusi online yang sela https t co ihtddjd3ra",
             "hal ini untuk kejar marketing slaes besar rp 6 triliun pada tahun ini https t co a0d59ei1bz","ojk nyata bahwa regulasi fintech lending di tanah air atur wajib tiap selenggara https t co eix0n8g9b9","berapa magnitudo gempa yang jadi https t co j2lz670tgh","ceo instagram tanggap duga bahwa aplikasi diam kuping suara cakap guna untuk sodor i https t co kbxm5lwfqk","jadi atlet tembak ini 6 pesona widi vierratale pegang senjata api https t co sqspwiv0o7","tak ada utang khusus hanya istana presiden bangun pakai apbn taucepattanpabatas uang ekonomi https t co triwzpgtd9","menko darmin bingung harga ayam anjlok taucepattanpabatas uang ekonomi ekonomi ekonomiindonesia https t co 8g8chh2bfh","meski begitu ezechiel tidak ikut lahap sesi latih sama rekan tim https t co l3uy22146m","turut bagaimana jika oposisi dapat jatah kursi di perintah tweet dapat dengan hashtag mencaripemimpin","sosok bayi laki temu tewas dekat komplek rumah karyawan bank indonesia kota malang polisi tengah https t co 7fsdylun8w","psis curi menang di markas badak lampung https t co eacewziyfd https t co 4kjkh0an6j","seru main tenis meja di dalam pasar grogol https t co a0cdygsmlg","<NAME> akan gelar konser 4 september 2019 di jiexpo mayor promotor niat undang joko widodo un https t co img1dwo0cj","main level neymar tak pantas main di prancis https t co mpwenrthkv","ojk akan patok tingkat kredit masalah fintech dana 1 persen https t co tr1ngg6lpf","jumlah massa yang ikut aksi di kawasan patung kuda minta kepada polisi agar habib bahar bebas mereka meng https t co ghdih5ofgt","lukman hakim saifuddin jalan sidang bagai saksi di adil tipikor https t co dclxvfnwry","video hakim konstitusi sudah punya putus sengketa pilpres https t co btjhai282d https t co 8jln2ayuez","psis curi poin penuh di kandang badak lampung https t co k3gzvmxqir","uang menag beri oleh syekh ibrahim atase agama dubes saudi untuk indone https t co 20bvimypy3","kualitas udara jakarta makin buruk airvisual juga saran jakartans untuk tidak laku giat di luar https t co rbu7nx4zxl","kpk panggil sofyan basir bagai saksi kasus suap layar https t co vbhpekgr8n","gak bisa libur ke pantai tenang ada rekomendasi waterpark di jakarta yang bikin libur kamu gak kalah seru kok https t co l7pf9aw1ew","rt vivabola bayern munich incar dembele rencana barcelona gaet neymar beranta https t co uttgrizwxn","rt vivalife ini yang akan laku vanessa angel telah keluar penjara https t co fvk6pr4mvx","senat izin rusia tangguh patuh pada janji nuklir https t co kyd2r2wfhy","telah putus mk ini yang akan laku kpu hingga jokowi-prabowo https t co 930vihlqpf via tribunnews","polda jawa barat tangkap dua duga teroris https t co 4xatlzttqu","rt kompasekonomi uangteman henti layan pinjam ini kata ojk https t co hqisex3vos https t co 729l83knic","penyalahgunaan edar narkoba kian masif turut data potensi desa 2018 bps olah lokadata sumbar jadi provi https t co 8jaowggh23","jokowi-kh ma ruf amin tak akan hadir sidang putus mk https t co bxadgycgjx","jelang putus mk tkn minta masyarakat tetap tenang aman jamin https t co ngggcei9kl https t co iyzcz7qnao","ngamplaglive kali ini bakal ada amigdala parahyena rozensky saddana tiur x yefta amp ega 28 juni di https t co mgvxnehnkt","populerb1 2 jk halalbihalal harus di tempat yang pantas bukan di mk https t co u6inavzsqz","populerb1 3 hakim konstitusi sudah putus sengketa pilpres 2019 https t co j9nrcwuf8f","populerb1 4 jk bamsoet dan aziz ingin jadi tum golkar https t co tqiaw1wimn","populerb1 5 3 nama ini nilai layak jadi calon menteri wakil orang asli papua https t co uixchxjpag","anies janji integrasi kendara umum https t co ikg2vectgb","populerb1 6 jelang putus mk ini rangkum sidang phpu pilpres https t co glxf9yeofw","populerb1 7 mr diy buka toko ke-100 di pluit village https t co nghztryzhn","populerb1 8 otto hasibuan praperadilan maki hadap jagung tidak tepat https t co aw6bdgauww","banyak informasi yang edar di sekitar kita namun belum tentu semua gizi cerna informasi sesuai yang anda https t co 5cpalrczls","produsen motor terus rilis produk baru meski pasar masih datar https t co 0p6zcptmuw otomotif","populerb1 9 istri ketua muhammadiyah padang lawas tewas tusuk rampok https t co cnfkogrlz9","operasi bina duduk https t co wz1gbbjyel","dinas pariwisata sulsel akan gelar pamer wisata khusus produk halal https t co f7bllste09","populerb1 10 kpk minta lukman hakim dan khofifah prioritas sidang kasus jual beli jabat https t co 9mauvdjfsx""conmebol kecam komentar lionel messi https t co bhazsu7msg","taksir juventus luka lebih pilih inter mil taucepattanpabatas bola sepakbola sepakboladunia boladunia https t co z2xbytrmdt","bagai reaksi atas menang as di final piala dunia putri https t co szzijvh4bv","peristiwa bakar ini tahu oleh tim layan dan lindung yanlin warga kjri jeddah https t co p76b8mqqoi","selamat pagi baca kompas nikmat liputantematis kena pajak digital yang saji mulai hari senin 8 7 https t co r4rfmnuadp","bos fifa dan presiden prancis cemooh dalam seremonial juara https t co fzg2k62wbm","<NAME>a 3-0 di stadion 30 juni kairo mesir senin 8 7 https t co hamowamf4j"]

vectorizer = TfidfVectorizer(stop_words='english')
X = vectorizer.fit_transform(documents)

true_k = 3
model = KMeans(n_clusters=true_k, init='k-means++', max_iter=100, n_init=1)
model.fit(X)

print("Top terms per cluster:")
order_centroids = model.cluster_centers_.argsort()[:, ::-1]
terms = vectorizer.get_feature_names()
for i in range(true_k):
    print("Cluster %d:" % i),
    for ind in order_centroids[i, :10]:
        print(' %s' % terms[ind]),
    print

print("\n")
print("Prediction")

Y = vectorizer.transform(["chrome browser to open."])
prediction = model.predict(Y)
print(prediction)

Y = vectorizer.transform(["My cat is hungry."])
prediction = model.predict(Y)
print(prediction)
pip install sklearn
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from sklearn.metrics import adjusted_rand_score
f= open("/gdrive/My Drive/MyFile3.txt","r") 
contents =f.read()
print(contents)
contents=[contents]

print(contents)

vectorizer = TfidfVectorizer(stop_words='english')
X = vectorizer.fit_transform(contents)

true_k = 3
model = KMeans(n_clusters=true_k, init='k-means++', max_iter=100, n_init=1)
model.fit(X)

print("Top terms per cluster:")
order_centroids = model.cluster_centers_.argsort()[:, ::-1]
terms = vectorizer.get_feature_names()
for i in range(true_k):
    print("Cluster %d:" % i),
    for ind in order_centroids[i, :10]:
        print(' %s' % terms[ind]),
    print

print("\n")
f.close()<jupyter_output>"china kian berburu aset nondolar as https://t.co/bnvftgy47f","kebakaran terjadi di universitas politeknik hong kong. diduga tempat persembunyian para demonstran itu terbakar aki… https://t.co/4viu8fsbd4","dari 46 orang tersebut, 23 orang merupakan jaringan sumatra utara-aceh, dengan empat orang di antaranya menyerahkan… https://t.co/b51qm9g844",""enggak bicara masalah posisi. kebetulan saya pernah (komisaris) di pln, jadi ditanya juga soal pengalaman saya wak… https://t.co/dzp2vltpeh","kemenag akan mendorong kebijakan berpihak kepada jamaah korban first travel. https://t.co/k2deddhus8","larangan masker dinilai tidak sesuai dengan konstitusi hong kong. https://t.co/vpekwjijy1","rt @korantempo: generasi millennial lebih suka menghabiskan uangnya untuk berlibur daripada melakukan investasi, seperti membeli rumah. ini…","ramalan zodiak cinta selasa 19 november 2019: keluarga leo ingin mengenal pasanganmu https://t.co/n8f5ohyqid","stefanos tsitsipas masih tak percaya bisa juarai atp finals[...]
{'fraction_non_alphanumeric': 0.08400801603206413, 'fraction_numerical': 0.018276553106212425, 'mean_word_length': 5.210054753608761, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '67014', 'license': 'MIT', 'n_tokens_mistral': 4789, 'n_tokens_neox': 4540, 'n_words': 1434, 'path': 'twitterb.ipynb', 'repo_name': 'adityabobbyr/dbstbi', 'size': '12623'}