<commit_before>from datetime import datetime
import traceback
import re

import pandas as pd
import nltk
from nltk.corpus import stopwords

from geeksrus import LOGGER
from geeksrus.utils.dbconn import read_mongo, write_mongo, read_mongo_projection, find_and_sort_desc
from geeksrus import dbcon
from geeksrus import cfg


stop = stopwords.words('english')
stop.extend(('.', ',', '"', "'", '?', '!', ':', ';',
             '(', ')', '[', ']', '{', '}','/','-', '@', '#','://',
             'https', 'RT', 'co', '&', '...'
             , 'say', 'says'))


class WordCloud:

    def getTokenFrequency(self, count_cutoff=2, min_length=3):
        try:
            data_df = read_mongo(dbcon, cfg['collection'])
            print(data_df.head())
            frequency_words_wo_stop = {}
            max = 1
            min = 1
            for data in data_df['text']:
                data = re.sub("http\S*\s*", ' ', data)
                data = re.sub("t.co\S*\s*", ' ', data)
                data = re.sub("bit.ly\S*\s*", ' ', data)
                tokens = nltk.wordpunct_tokenize(data)
                for token in tokens:
                    token = token.strip()
                    if len(token) > 0 and token.lower() not in stop:
                        if token in frequency_words_wo_stop:
                            count = frequency_words_wo_stop[token]
                            count = count + 1

                            if count > max:
                                max = count

                            if count < min:
                                min = count

                            frequency_words_wo_stop[token] = count
                        else:
                            frequency_words_wo_stop[token] = 1
            wordcloud= []

            if max == min:
                max = max + 1

            for key, value in frequency_words_wo_stop.items():
                word_freq = {}
                if value > count_cutoff and len(key) >= min_length:
                    word_freq['text'] = key
                    normalized_value = ((max - value)/ (max - min)) * 100.0
                    word_freq['size'] = normalized_value
                    wordcloud.append(word_freq)

            print(wordcloud)


            # {'text': 'word', 'size' : count}
            mongo_doc = {}
            mongo_doc['word_cloud'] = wordcloud
            mongo_doc['timestamp'] = datetime.now()

            write_mongo(db_conn=dbcon, collection=cfg['word_cloud_collection'], document=mongo_doc)

        except Exception as e:
            LOGGER.error(traceback.format_exc())

    def getMentionsFrequency(self):
        try:
            projection = []
            projection.append('entities.user_mentions')
            data = read_mongo_projection(dbcon, cfg['collection'], query=projection)
            data.to_csv('trials.csv')
            normalized_df = pd.io.json.json_normalize(data.entities, 'user_mentions')


            grouped_df = normalized_df.groupby(by=['screen_name'], as_index=False)['id'].count()
            grouped_df_screen_count = grouped_df[['screen_name','id']]
            grouped_df_screen_count = grouped_df_screen_count.sort(columns='id', ascending=False).reset_index()
            grouped_df_screen_count = grouped_df_screen_count.ix[:20,]

            max = grouped_df_screen_count.id.max()
            min = grouped_df_screen_count.id.min()

            count_data = []
            for row in grouped_df_screen_count.iterrows():
                screen_count = {}
                screen_count["text"] = row[1]['screen_name']
                normalized_value = ((max - row[1]['id'])/ (max - min)) * 100.0
                screen_count["size"] = normalized_value
                count_data.append(screen_count)

            mongo_doc = {}
            mongo_doc['word_cloud'] = count_data
            mongo_doc['timestamp'] = datetime.now()
            print(mongo_doc)

            write_mongo(db_conn=dbcon, collection=cfg['mentions_collection'], document=mongo_doc)

        except Exception as e:
            LOGGER.error(traceback.format_exc())

    def getMaximumUserCount(self):
        try:
            projection = []
            projection.append('user.screen_name')
            projection.append('user.id')
            data = read_mongo_projection(dbcon, cfg['collection'], query=projection)
            column_df = data.apply(self.generate_columns_from_dict, axis=1)
            grouped_df = column_df.groupby(by=['screen_name'], as_index=False)['id'].count()
            grouped_df_screen_count = grouped_df[['screen_name','id']]
            grouped_df_screen_count = grouped_df_screen_count.sort(columns='id', ascending=False).reset_index()
            grouped_df_screen_count = grouped_df_screen_count.ix[:20,]
            count_data = []


            max = grouped_df_screen_count.id.max()
            min = grouped_df_screen_count.id.min()

            for row in grouped_df_screen_count.iterrows():
                screen_count = {}
                screen_count["text"] = row[1]['screen_name']
                normalized_value = ((max - row[1]['id'])/ (max - min)) * 100.0
                screen_count["size"] = normalized_value
                count_data.append(screen_count)

            mongo_doc = {}
            mongo_doc['word_cloud'] = count_data
            mongo_doc['timestamp'] = datetime.now()

            write_mongo(db_conn=dbcon, collection=cfg['users_collection'], document=mongo_doc)

        except Exception as e:
            LOGGER.error(traceback.format_exc())

    def generate_columns_from_dict(self, row):<commit_msg>Taking care of punctuations in text<commit_after>from datetime import datetime
import traceback
import re

import pandas as pd
import nltk
from nltk.corpus import stopwords

from geeksrus import LOGGER
from geeksrus.utils.dbconn import read_mongo, write_mongo, read_mongo_projection, find_and_sort_desc
from geeksrus import dbcon
from geeksrus import cfg


stop = stopwords.words('english')
stop.extend(('.', ',', '"', "'", '?', '!', ':', ';',
             '(', ')', '[', ']', '{', '}','/','-', '@', '#','://',
             'https', 'RT', 'co', '&', '...'
             , 'say', 'says'))


class WordCloud:

    def getTokenFrequency(self, count_cutoff=2, min_length=3):
        try:
            data_df = read_mongo(dbcon, cfg['collection'])
            print(data_df.head())
            frequency_words_wo_stop = {}
            max = 1
            min = 1
            for data in data_df['text']:
                data = re.sub("http\S*\s*", ' ', data)
                data = re.sub("t.co\S*\s*", ' ', data)
                data = re.sub("bit.ly\S*\s*", ' ', data)
                data = re.sub("\W", ' ', data)
                tokens = nltk.wordpunct_tokenize(data)
                for token in tokens:
                    token = token.strip()
                    if len(token) > 0 and token.lower() not in stop:
                        if token in frequency_words_wo_stop:
                            count = frequency_words_wo_stop[token]
                            count = count + 1

                            if count > max:
                                max = count

                            if count < min:
                                min = count

                            frequency_words_wo_stop[token] = count
                        else:
                            frequency_words_wo_stop[token] = 1
            wordcloud= []

            if max == min:
                max = max + 1

            for key, value in frequency_words_wo_stop.items():
                word_freq = {}
                if value > count_cutoff and len(key) >= min_length:
                    word_freq['text'] = key
                    normalized_value = ((max - value)/ (max - min)) * 100.0
                    word_freq['size'] = normalized_value
                    wordcloud.append(word_freq)


            # {'text': 'word', 'size' : count}
            mongo_doc = {}
            mongo_doc['word_cloud'] = wordcloud
            mongo_doc['timestamp'] = datetime.now()

            write_mongo(db_conn=dbcon, collection=cfg['word_cloud_collection'], document=mongo_doc)

        except Exception as e:
            LOGGER.error(traceback.format_exc())

    def getMentionsFrequency(self):
        try:
            projection = []
            projection.append('entities.user_mentions')
            data = read_mongo_projection(dbcon, cfg['collection'], query=projection)
            data.to_csv('trials.csv')
            normalized_df = pd.io.json.json_normalize(data.entities, 'user_mentions')


            grouped_df = normalized_df.groupby(by=['screen_name'], as_index=False)['id'].count()
            grouped_df_screen_count = grouped_df[['screen_name','id']]
            grouped_df_screen_count = grouped_df_screen_count.sort(columns='id', ascending=False).reset_index()
            grouped_df_screen_count = grouped_df_screen_count.ix[:20,]

            max = grouped_df_screen_count.id.max()
            min = grouped_df_screen_count.id.min()

            count_data = []
            for row in grouped_df_screen_count.iterrows():
                screen_count = {}
                screen_count["text"] = "@" + row[1]['screen_name']
                normalized_value = ((max - row[1]['id'])/ (max - min)) * 100.0
                screen_count["size"] = normalized_value
                count_data.append(screen_count)

            mongo_doc = {}
            mongo_doc['word_cloud'] = count_data
            mongo_doc['timestamp'] = datetime.now()

            write_mongo(db_conn=dbcon, collection=cfg['mentions_collection'], document=mongo_doc)

        except Exception as e:
            LOGGER.error(traceback.format_exc())

    def getMaximumUserCount(self):
        try:
            projection = []
            projection.append('user.screen_name')
            projection.append('user.id')
            data = read_mongo_projection(dbcon, cfg['collection'], query=projection)
            column_df = data.apply(self.generate_columns_from_dict, axis=1)
            grouped_df = column_df.groupby(by=['screen_name'], as_index=False)['id'].count()
            grouped_df_screen_count = grouped_df[['screen_name','id']]
            grouped_df_screen_count = grouped_df_screen_count.sort(columns='id', ascending=False).reset_index()
            grouped_df_screen_count = grouped_df_screen_count.ix[:20,]
            count_data = []


            max = grouped_df_screen_count.id.max()
            min = grouped_df_screen_count.id.min()

            for row in grouped_df_screen_count.iterrows():
                screen_count = {}
                screen_count["text"] = "@" + row[1]['screen_name']
                normalized_value = ((max - row[1]['id'])/ (max - min)) * 100.0
                screen_count["size"] = normalized_value
                count_data.append(screen_count)

            mongo_doc = {}
            mongo_doc['word_cloud'] = count_data
            mongo_doc['timestamp'] = datetime.now()

            write_mongo(db_conn=dbcon, collection=cfg['users_collection'], document=mongo_doc)

        except Exception as e:
            LOGGER.error(traceback.format_exc())

    def generate_columns_from_dict(self, row):
{'fraction_non_alphanumeric': 0.12260298295454546, 'fraction_numerical': 0.005149147727272727, 'mean_word_length': 1.8996138996138996, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '644454', 'n_tokens_mistral': 3372, 'n_tokens_neox': 3212, 'n_words': 565}