<jupyter_start><jupyter_text>Social Media Analytics (SMA)https://taudata.blogspot.comSMA-01: Digital Data Gatheringhttps://taudata.blogspot.com/2022/04/sma-01.html(C) <NAME><jupyter_code>import warnings; warnings.simplefilter('ignore')
import nltk

try:
    import google.colab
    IN_COLAB = True
    !wget https://raw.githubusercontent.com/taudata-indonesia/eLearning/master/taudataDDGsna.py
    !mkdir data
    !wget -P data/ https://raw.githubusercontent.com/taudata-indonesia/eLearning/master/data/contoh.pdf
    !wget -P data/ https://raw.githubusercontent.com/taudata-indonesia/eLearning/master/data/slang.txt
    !wget -P data/ https://raw.githubusercontent.com/taudata-indonesia/eLearning/master/data/stopwords_id.txt
    !wget -P data/ https://raw.githubusercontent.com/taudata-indonesia/eLearning/master/data/stopwords_en.txt
    !wget -P data/ https://raw.githubusercontent.com/taudata-indonesia/eLearning/master/data/kata_dasar.txt
    !wget -P data/ https://raw.githubusercontent.com/taudata-indonesia/eLearning/master/data/wn-ind-def.tab
    !wget -P data/ https://raw.githubusercontent.com/taudata-indonesia/eLearning/master/data/wn-msa-all.tab
    !wget -P data/ https://raw.githubusercontent.com/taudata-indonesia/eLearning/master/data/ind_SA.csv
    !wget -P data/ https://raw.githubusercontent.com/taudata-indonesia/eLearning/master/data/all_indo_man_tag_corpus_model.crf.tagger

    !pip install --upgrade spacy python-crfsuite unidecode textblob sastrawi sklearn-pycrfsuite
    !pip install --upgrade unidecode twython tweepy beautifulsoup4 tika
    !python -m spacy download en
    !python -m spacy download xx
    !python -m spacy download en_core_web_sm
    nltk.download('popular')
except:
    IN_COLAB = False
    print("Running the code locally, please make sure all the python module versions agree with colab environment and all data/assets downloaded")
import taudataDDGsna as tau
import tweepy, json, urllib.request, requests
from urllib.request import Request, urlopen
from twython import TwythonStreamer
from bs4 import BeautifulSoup as bs
from tqdm import tqdm
"Done"<jupyter_output><empty_output><jupyter_text>Aturan, bentuk data, &amp; error codes twitter		https://dev.twitter.com/rest/public/rate-limiting			https://dev.twitter.com/overview/terms/agreement-and-policy			https://dev.twitter.com/overview/api/response-codes			https://dev.twitter.com/overview/api/tweets<jupyter_code># Contoh API Keys (Sesuaikan dengan API keys masing-masing)
Ck = '' # consumer_key
Cs = '' # consumer_secret
At = '-' # access_token
As = '' # access_secret

'Done'
import tweepy

auth = tweepy.auth.OAuthHandler(Ck, Cs)
auth.set_access_token(At, As)
api = tweepy.API(auth, wait_on_rate_limit=True, timeout=180, retry_count=5, retry_delay=3)
usr_ = api.verify_credentials()
print('Welcome "{}" you are now connected to twitter server'.format(usr_.name))
# Kita bisa post, delete, follow, like, reply, retweet, dsb.
# WARNING ... Status harus dibedakan, jika tidak akan gagal karena twitter mendeteksi status duplikat.

status = "Testing Digital Data Gathering - kuliah EDA-Viz"
t = api.update_status(status)
# Silahkan cek timeline setelah ini
"Done! ... Silahkan cek timeline twitter anda."
qry = 'minyak goreng'
N = 1000 # jumlah N tweet "terkini" yang ingin diambil
T = []
for post in tqdm(tweepy.Cursor(api.search_tweets, q=qry, lang='id', tweet_mode='extended').items(N)):
    T.append(post)
    
"Done"
T[0]._json
# T bukan sekedar list of tweets 
tweet = [t._json for t in T]
print(' Berhasil mendapatkan {} tweets'.format(len(tweet)))
# Data pertama
tweet[0]
# Datanya berbentuk JSON
tweet[0]['created_at'], tweet[0]['full_text']
# Contoh mengakses data spesifik pada tweet yang pertama:
print('tweet pertama oleh "{}" : "{}"'.format(tweet[0]['user']['screen_name'],tweet[0]['full_text']))<jupyter_output><empty_output><jupyter_text>Menyimpan hasil crawling (sederhana ke Text file berformat json)<jupyter_code>import json 
def saveTweets(tweets, file='Tweets.json'): #in Json Format
    with open(file, 'w') as f:
        for t in tweets:
            try:
                f.write(json.dumps(t)+'\n')
            except:
                pass
# Menyimpan hasil crawling twitter
fileName = 'Tweets_2021.json'
saveTweets(tweet, file=fileName)
print('Saved to '+fileName)<jupyter_output><empty_output><jupyter_text>Load Kembali?<jupyter_code>def loadTweets(file='Tweets.json'):
    f=open(file,encoding='utf-8', errors ='ignore', mode='r')
    T=f.readlines();f.close()
    for i,t in enumerate(T):
        T[i] = json.loads(t.strip())
    return T
# Me-load kembali jika (misal) analisa ingin dilakukan di lain waktu
# Sengaja nama variabelnya saya bedakan (T2)
T2 = loadTweets(file=fileName)
print('tweet pertama oleh "{}" : "{}"'.format(T2[0]['user']['screen_name'],T2[0]['full_text']))
# Contoh mengambil hanya data tweet
D = [t['full_text'] for t in T2]
D[:5] # 5 tweet pertama
# Mari kita coba #1
topic = 'from:jokowi'
T = api.search_tweets(q=topic, lang='id', tweet_mode = 'extended')
tweet = [t._json for t in T]
isiTweet = [t['full_text'] for t in tweet]
isiTweet
# https://www.latlong.net/convert-address-to-lat-long.html

alamat = "jl. Ir. H. Juanda No 95 tangerang selatan ciputat"
lat = '-6.307870'
lon = '106.755820'

"Done"
# Mari kita coba #3 gunakan google (map) untuk koordinat suatu lokasi
# http://thoughtfaucet.com/search-twitter-by-location/
# misal search tweet tentang "makanan" di Depok dan sekitarnya
radius = '100km'
Geo = ','.join([lat, lon, radius])
qry = 'beli mobil'
for tweet_ in tweepy.Cursor(api.search_tweets, q=qry, geocode=Geo).items(N):
    print([tweet_.created_at, tweet_.text.encode('utf-8'), tweet_.user.id, tweet_.geo])
# Streaming tweets. Untuk percobaan pilih topicS sesuatu yg sedang trending/populer "saat ini".
# Atau bisa coba dengan mengirim tweet sendiri :)
from twython import TwythonStreamer

def streamTwitter(topicS, lang):
    class MyStreamer(TwythonStreamer):
        def on_success(self, data):
            global count
            count+=1
            print('tweet from {}, post: {}'.format(data['user']['screen_name'], data['text']))
            if count==maxTweet:
                print('\nFinished streaming %.0f tweets' %(maxTweet)); self.disconnect()
        def on_error(self, status_code, data):
            print('Error Status = %s' %status_code); self.disconnect()

    while count<maxTweet:
        stream = MyStreamer(Ck, Cs, At, As)
        stream.statuses.filter(track=topicS)
maxTweet, count = 5, 0 # Rubah sesuai dengan kebutuhan, Untuk percobaan ini cukup (misal) 12 tweet
lan = set(['en','id']) # bahasa bisa dipilih > 1
topicS = ['taudata', 'ui'] # Bisa>1

streamTwitter(topicS, lan)
# preprocess "D": kumpulan tweet
slangFixId = tau.loadCorpus(file = 'data/slang.txt', sep=':')
stopId, _ = tau.LoadStopWords(lang='id')
stopId.add("rt")

list(stopId)[:5]
from tqdm import tqdm 

cleanD = []
for t in tqdm(D):
    doc = tau.cleanText(t, fix=slangFixId, lan='id', stops = stopId)
    cleanD.append(doc)
    
"Done"
print(D[0], end='\n\n')
print(cleanD[0], end='\n\n')
# Save ke txt ... lalu nanti akan di upload ke Voyant Tools
filename = 'tweets_2021.txt'
with open(filename, 'w') as f:
    for T in cleanD:
        f.write(T+'\n')
        
"Tweets Saved!"<jupyter_output><empty_output><jupyter_text>HashTag Analysis* Frequency-Based* Bisa ditambahkan analisa hashtags analysis ini dari waktu ke waktu<jupyter_code>import re

HTfilters = set(['zz', 'architec', 'prize', 'stirli', 'architect', 'london', 'cpd', 'design', 'stirling', 'photogr', 'gemini', 
                 'mule', 'karaoke', 'playing', 'official', 'berita', 'follow', 'retweet', 'mufc', 'ntms', 'infolimit', 'eeaa', 
                 'eaa', 'cfc', 'caprico', 'breaking','news', 'libra', 'mereka', 'brankas', 'psikolog', 'aquarius', 'klc'])
# modifikasi HTfilters sesuai data kamu
HT = {'hashtags':[]}
count = 0
getHashTags = re.compile(r"#(\w+)")
for i, d in tqdm(enumerate(D)):
    hashtags = re.findall(getHashTags, d)
    if hashtags:
        TG = []
        for tag in hashtags:
            dTag = str(tag).strip().lower()
            if len(dTag)>2:
                add = True
                for f in HTfilters:
                    if f in dTag:
                        add=False; break
                if add:
                    TG.append('#'+dTag); count += 1
            HT['hashtags'].append(TG)
print('Found {} number of hashtags'.format(count))
import pandas as pd

N = 50 # Number of top frequent hashtags to be plotted

dtHT = [x for t in tqdm(HT['hashtags']) for x in t] # any(h not in x for h in HTfilters)
dtHT = pd.Series(dtHT)
dtHT = dtHT.value_counts()
dtHT = dtHT.sort_index()
dtHT = dtHT.sort_values(ascending = False) 
dtHT.to_csv('hashTags_2021.csv', encoding='utf8')
dtHT = dtHT.iloc[:N]

p = dtHT.plot(kind='barh', figsize=(12,8), legend = False)<jupyter_output><empty_output><jupyter_text>Web Scrapping?<jupyter_code>import urllib.request
from bs4 import BeautifulSoup as bs

URL = 'https://www.beritasatu.com/beritasatu-tv'
Doc = urllib.request.urlopen(URL).read()
Doc = bs(Doc,'lxml').text
print(Doc)<jupyter_output><empty_output><jupyter_text>Loading Local Documents1. Instalasi Java - JDK 8 ... ingat harus JDK 8 - https://www.oracle.com/id/java/technologies/javase/javase-jdk8-downloads.html - Set Java Home Directory2. Instalasi Tika Server : - Download Tika App Executable Java Jar: https://archive.apache.org/dist/tika/tika-app-1.24.1.jar - Put in Python home directory3. Instalasi Module Python Tika4. Reading pdf Files (Python Code below) Tika can read Pdf, DocX, PPTX, etc.<jupyter_code>from tika import parser#, unpack

def readDocs(file):
    if 'pdf' in file:
        headers = {'X-Tika-PDFextractInlineImages': 'true',} 
        raw = parser.from_file(file, headers=headers)
    else:
        raw = parser.from_file(file)
    if 'content' in raw.keys():
        return raw['content']
    else:
        return None
try:
    doc = readDocs('data/contoh.pdf')
except:
    !mkdir data
    !wget -P data/ https://raw.githubusercontent.com/taudata-indonesia/eLearning/master/data/contoh.pdf
    doc = readDocs('data/contoh.pdf')
    
print(doc)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13877149877149877, 'fraction_numerical': 0.010319410319410319, 'mean_word_length': 4.002949852507375, 'pattern_counts': {'":': 1, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 21, 'lorem ipsum': 0, 'www.': 3, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '528404', 'license': 'Apache-2.0', 'n_tokens_mistral': 3897, 'n_tokens_neox': 3678, 'n_words': 864, 'path': 'sma-01-codes.ipynb', 'repo_name': 'taudata-indonesia/elearning', 'size': '10183'}