<jupyter_start><jupyter_text>MongoDB playing with Tags in python<jupyter_code>%matplotlib inline
import pymongo
from pymongo import MongoClient
from pymongo.errors import ConnectionFailure
from bson import json_util, ObjectId
import pandas as pd
from pandas import DataFrame
from pandas.io.json import json_normalize
import numpy as np
import requests
import json, os
import configparser
import matplotlib.pyplot as plt
import numpy as np
from scipy import stats
import seaborn as sns
import warnings
import random
import pprint
from datetime import datetime
random.seed(datetime.now())
warnings.filterwarnings('ignore')

# Make plots larger
plt.rcParams['figure.figsize'] = (10, 6)
client = MongoClient('localhost', 27017)
db=client.tweets
db.collection_names(include_system_collections=False)
try:
    result = db.tweets.drop()
    print ("analytics tweets dropped")
except:
    pass
def load_json(j):
    p=os.path.join("data/", j)
    print (p)
    with open(p, 'rU') as f:
      data = [json.loads(row) for row in f]
    return data 
tweets_j=load_json('db_tweets.json')
tweets_j[0]
print(tweets_j[0]['text'])
def tweet_json(tid,text,created,favorite_count,retweet_count,urls,tags):
    j={
    "tweet_id" : tid,
    "text" : text,
    "favorite_count" : favorite_count,  
    "retweet_count" : retweet_count,
    "urls" : urls, 
    "tags" : tags,         
    "created_at" : created}
    return j
# Twitter dates are of the form Sun Mar 15 21:41:54 +0000 2015
datestrings=['Sun Mar 15 21:41:54 +0000 2015','Tue Mar 29 08:11:25 +0000 2011']
from datetime import timedelta
from email.utils import parsedate_tz
from dateutil.parser import parse

def to_datetime(datestring):
    time_tuple = parsedate_tz(datestring.strip())
    dt = datetime(*time_tuple[:6])
    return dt - timedelta(seconds=time_tuple[-1])

ts=to_datetime(datestrings[0])
print (ts.strftime("%Y-%m-%d %H:%M"))
ts=to_datetime(datestrings[1])
print (ts.strftime("%Y-%m-%d %H:%M"))
hashtags={}
starter_tags=['Bigdata','big data','algorithm','big data','AI','MongoDB','SQL','artificial intelligence','machine learning']
for tag in starter_tags:
    hashtags[tag]=0
urls={}
tags={}
print(hashtags)
sample_tweet_text="RT @TheHesterView Tutorials on big data, big data, AI, MongoDB, SQL, artificial intelligence, machine learning. hackathons, crowdsourcing, #bigdata http://t.co/6HWjCv3BL5 Lets join "
print (sample_tweet_text)
import re
def tokenize(txt):  
  txt=re.sub(r'\n', ' ',txt)
  txt=re.compile(r'[\.][ ]+').sub(' ',txt)      
  txt=re.compile(r'[\,][ ]+').sub(' ',txt)    
  txt=re.compile(r'[_+;=!@$%^&\*\"\?]').sub(' ',txt)  
  splitter=re.compile(r'[ ]+')
  # Split the words by non-alpha characters
  words=splitter.split(txt)
  return words
print (tokenize(sample_tweet_text))
s='http://t.co/6HWjCv3BL5'
print (s[0:4].lower())
from nltk.corpus import stopwords
stop_words_list = list(stopwords.words('english'))
stop_words={}
for tag in stop_words_list:
    stop_words[tag]=0
print (stop_words.keys())
def update_urls_tags(url_list,urls,hashtag_list,hashtags,tag_list,tags):  
  for url in url_list:
      if url in urls:
        urls[url]=urls[url]+1
      else:
        urls[url]=1	 
  for tag in tag_list:
      if tag in tags:
        tags[tag]=tags[tag]+1
      else:
        tags[tag]=1	 
  for hashtag in hashtag_list:
      if hashtag in hashtags:
        hashtags[hashtag]=hashtags[hashtag]+1
      else:
        hashtags[hashtag]=1        
  return urls,hashtags,tags
def extract_tags_urls(dct,words,stop):
  i=0
  tags={} 
  tokens={}     
  urls={}     
  size=len(words)   
  while i < size:
    ngram = words[i]
    i=i+1
    if len(ngram) < 1: continue
    if len(ngram) > 4:        
      if ngram[0:4].lower()=='http':
        if ngram in urls:
          urls[ngram]=urls[ngram]+1
        else:
          urls[ngram]=1	 
    if ngram[0]=='#':
  #    ngram=re.sub(r'\#', '',ngram)     if you want to remove the # 
      tags[ngram]=1 
    if ngram.lower() not in stop:    
        tokens[ngram]=1	          
    if ngram in dct:
      tags[ngram]=1
    if i < (size-1):
      ngram = words[i] + ' ' + words[i+1]
      if words[i].lower() not in stop:    
        tokens[ngram]=1	        
      if ngram in dct:
        tags[ngram]=1
    if i < (size-2):
      ngram = words[i] + ' ' + words[i+1] + ' ' + words[i+2]           
      if ngram in dct:
        tags[ngram]=1
  return list(tags.keys()),list(urls.keys()),list(tokens.keys())
print (extract_tags_urls(hashtags,(tokenize(sample_tweet_text)),stop_words))
cnt=0
for tweet in tweets_j:
    ts=datetime.now()
    try:
        ts=to_datetime(tweet['created_at'])
    except:
        continue  
    favorite_count=0
    try:
        favorite_count=int(tweet['favorite_count'])
    except:
        pass 
    retweet_count=0
    try:
        retweet_count=int(tweet['retweet_count'])
    except:
        pass  
    tweet_tags,tweet_urls,tweet_ngrams=extract_tags_urls(hashtags,(tokenize(tweet['text'])),stop_words)
    urls,hashtags,tags=update_urls_tags(tweet_urls,urls,tweet_tags,hashtags,tweet_ngrams,tags)
    try:
        j=tweet_json(tweet['id'],tweet['text'],ts,favorite_count,retweet_count,tweet_urls,tweet_tags)
        result = db.tweets.insert_one(j)
        cnt+=1
    except:
        pass       
print ("%d tweets inserted."%cnt)
for tweet in db.tweets.find().limit(9):
    pprint.pprint(tweet)
for key, value in hashtags.items():
    print ("%s count %d"%(key, value))
def trim_dct(dct,thresh): 
  tmp={}    
  keys=dct.keys()
  for key in keys:
    if dct[key] > thresh:
        tmp[key]=dct[key]
  return tmp
'''
for key, value in tags.items():
    print ("%s count %d"%(key, value))
'''    
tags=trim_dct(tags,3)
for key, value in tags.items():
    print ("%s count %d"%(key, value))
for key, value in urls.items():
    print ("%s count %d"%(key, value))<jupyter_output>http://t.co/gLf50KtZKx count 1
http://t.co/MiEIZmKzEb count 1
http://t.co/i37gbFthQw count 1
http://t.co/uYQIXOSMHs count 1
http://t.co/uQSGCKEiOn count 1
http://t.co/6HWjCv3BL5 count 1
http://t.co/rLNk2UtnkL count 1
http://t.co/92XV9CUUpO count 1
http://t.co/fi0MuG9QUH count 1
http://t.co/jRLp7… count 1
http://t.co/w4xw93Tw19 count 2
http://t.co/wdK8JZM7db count 1
http://t.co/tkSfjPpods count 1
http://t.co/mM0… count 1
http… count 1
https://t.co/BGmnXAOlIY count 1
http:… count 1
http://t.co/EBChICLch2 count 1
http://t.co/SgcfxbdQvZ count 1
http://t.c… count 1
http://t.co/Jfh5Xc2hQz count 1
http://t.co/SIHVND8mDY count 2
http://t.co/… count 2
http://t.co/OGcvtI1663 count 1
http://t.co/n5kJRoYydW count 1
https://t.co/TzrPR5rtY8 count 1
http://t.co/UQg7q3EvHp count 1
http://t.co/BZN44MeEYF count 2
http://t.co/rPVg5Kd7q1 count 1
http://t.co/gXAsZVt1IQ count 1
http://t.co/Qh5Lkvcyyz count 1
http://t.co/d6O7BOcfKQ count 2
http://t.co/CUES54iN0m count 1
http://t.co/Oq7Dkb4TRy count 1
http://[...]
{'fraction_non_alphanumeric': 0.14805612394036832, 'fraction_numerical': 0.027331189710610933, 'mean_word_length': 3.507905138339921, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '144763', 'license': 'MIT', 'n_tokens_mistral': 2814, 'n_tokens_neox': 2690, 'n_words': 582, 'path': "Prof. Nik Brown's Notes/D06. MongoDB/6. MongoDB_python_Tags.ipynb", 'repo_name': 'PratikMahajan/Data-Science-Reference', 'size': '6842'}