<commit_before>import pymongo
import numpy
from collections import Counter


class YoutubeStats:
    def __init__(self):
        self.conn = pymongo.MongoClient()
        self.db = self.conn['trendingapps']
        self.ycollec = self.db['youtube']
        self.query = self.ycollec.find({}, {'_id': 0})

    def get_lists(self):
        comment_count_dict = {}
        likes_dict = {}
        views_dict = {}
      
        for count in self.query:        
            comments_count = []
            likes_list = []
            views_list = []
            for x in range(len(count)-1):
                comments_count.append(int(count['video_' + str(x)]['comment_count']))
                likes_list.append(int(count['video_' + str(x)]['likes']))
                views_list.append(int(count['video_' + str(x)]['views']))
       
            comments_count.sort()
            likes_list.sort()
            views_list.sort()
       
            comment_count_dict[str(count['appName'])] = comments_count
            likes_dict[str(count['appName'])] = likes_list
            views_dict[str(count['appName'])] = views_list
   
        return comment_count_dict, likes_dict, views_dict
    
    def find_mean(self, comment_dict, like_dict, view_dict):    
        comment_mean = {}
        like_mean = {}
        view_mean = {}
       
        for (key, value) in comment_dict.items():
            mean = numpy.mean(value)
            comment_mean[key] = round(mean, 1)
       
        for (key2, value2) in like_dict.items():         
            mean = numpy.mean(value2)
            like_mean[key2] = round(mean, 1)
       
        for (key3, value3) in view_dict.items():         
            mean = numpy.mean(value3)
            view_mean[key3] = round(mean, 1)
       
        return comment_mean, like_mean, view_mean    

    def find_median(self, comment_dict, like_dict, view_dict):
        comment_median = {}
        like_median = {}
        view_median = {}    
        
        for (app1, comment_count_list) in comment_dict.items():
            median1 = numpy.median(comment_count_list)
            comment_median[app1] = median1 
    
        for (app2, likes_list) in like_dict.items():
            median2 = numpy.median(likes_list)
            like_median[app2] = median2 
            
        for (app3, view_list) in view_dict.items():
            median3 = numpy.median(view_list)
            view_median[app3] = median3 
                
        return comment_median, like_median, view_median

    def find_mode(self, comment_dict, like_dict, view_dict):
        comment_mode = {}
        likes_mode = {}    
        views_mode = {}    
        for (app, comment_lists) in comment_dict.items():
            x = Counter(comment_lists).most_common(1)   
            comment_mode[app] = x[0][0]

        for (app, likes_lists) in like_dict.items():
            y = Counter(likes_lists).most_common(1)   
            likes_mode[app] = y[0][0]
    
        for (app, views_lists) in view_dict.items():
            z = Counter(views_lists).most_common(1)   
            views_mode[app] = z[0][0]
    
        return comment_mode, likes_mode, views_mode
    
    def find_standard_deviation(self, comment_dict, like_dict, view_dict):
        comment_sd = {}
        like_sd = {}
        view_sd = {}
    
        for (app, comment_lists) in comment_dict.items():
            sd = numpy.std(comment_lists)    
            comment_sd[app] = round(sd, 1)
    
        for (app, likes_lists) in like_dict.items():
            sd = numpy.std(likes_lists)
            like_sd[app] = round(sd, 1)
    
        for (app, views_lists) in view_dict.items():
            sd = numpy.std(views_lists) 
            view_sd[app] = round(sd, 1)
        
        return comment_sd, like_sd, view_sd

    def stats(self):
        comment_dict, like_dict, view_dict = self.get_lists()
   
        comment_mean, like_mean, view_mean = self.find_mean(comment_dict, like_dict, view_dict)
        comment_median, like_median, view_median = self.find_median(comment_dict, like_dict, view_dict)    
        comment_mode, like_mode, view_mode = self.find_mode(comment_dict, like_dict, view_dict)
        comment_sd, like_sd, view_sd = self.find_standard_deviation(comment_dict, like_dict, view_dict)
       
        stats_dict = {}        
        stats_dict['mean'] = {'comment_count': comment_mean, 'likes': like_mean, 'views': view_mean}      
        stats_dict['median'] = {'comment_count': comment_median, 'likes': like_median, 'views': view_median}  
        stats_dict['mode'] = {'comment_count': comment_mode, 'likes': like_mode, 'views': view_mode}
        stats_dict['standard_deviation'] = {'comment_count': comment_sd, 'likes': like_sd, 'views': view_sd}
   
        return stats_dict<commit_msg>sentimental analysis<commit_after>import pymongo
import numpy
from collections import Counter
from textblob import TextBlob

class YoutubeStats:
    def __init__(self):
        self.conn = pymongo.MongoClient()
        self.db = self.conn['trendingapps']
        self.ycollec = self.db['youtube']
        self.query = self.ycollec.find({}, {'_id': 0})

    def get_lists(self):
        clw_list = []
        for count in self.query:        
            comments_count = []
            likes_list = []
            views_list = []
            comment_lists = []
            comments = {}            
            for x in range(len(count)-1):
                comments_count.append(int(count['video_' + str(x)]['comment_count']))
                likes_list.append(int(count['video_' + str(x)]['likes']))
                views_list.append(int(count['video_' + str(x)]['views']))
                
                if int(count['video_' + str(x)]['comment_count']) != 0:            
                    for comment in range(len(count['video_' + str(x)]['comments'])):
                        comment_str = count['video_' + str(x)]['comments']['comment_' + str(comment)].encode('ascii', 'ignore')
                        if comment_str != "":
                            if comment_str.isspace() == False:                       
                                comment_lists.append(comment_str)
                           
            comments_count.sort()
            likes_list.sort()
            views_list.sort()
            
            app_dict = {}
            app_dict['appName'] = count['appName']
            app_dict['comment_count'] = comments_count            
            app_dict['likes'] = likes_list
            app_dict['views'] = views_list
            app_dict['comments_sent'] = comment_lists
            clw_list.append(app_dict)
   
        return clw_list
     
    def get_stats(self):
        clw_list = self.get_lists()  
        stats_list = []
        
        for app in clw_list:
            app_stats = {}
            app_stats['appName'] = app['appName']
            
            comment_count_list = app['comment_count']
            comment_mean = round(numpy.mean(comment_count_list), 1)
            comment_median = numpy.median(comment_count_list)
            x = Counter(comment_count_list).most_common(1)            
            comment_mode = x[0][0]  
            comment_sd = round(numpy.std(comment_count_list), 1)
            app_stats['comment_count'] = {'mean': comment_mean, 'median': comment_median, 'mode': comment_mode, 'standard_deviation': comment_sd}
            
            likes_list = app['likes']
            likes_mean = round(numpy.mean(likes_list), 1)
            likes_median = numpy.median(likes_list)
            y = Counter(likes_list).most_common(1)            
            likes_mode = y[0][0]  
            likes_sd = round(numpy.std(likes_list), 1)
            app_stats['likes'] = {'mean': likes_mean, 'median': likes_median, 'mode': likes_mode, 'standard_deviation': likes_sd}
            
            views_list = app['views']
            views_mean = round(numpy.mean(views_list), 1)
            views_median = numpy.median(views_list)
            z = Counter(views_list).most_common(1)            
            views_mode = z[0][0]  
            views_sd = round(numpy.std(views_list), 1)
            app_stats['views'] = {'mean': views_mean, 'median': views_median, 'mode': views_mode, 'standard_deviation': views_sd}
            
            comments_polarity = []
            comments = {}
            comments_pol = {}
            comment_list = app['comments_sent']
            for comment in comment_list:
                sent_analysis = TextBlob(comment)    
                if sent_analysis.sentiment.polarity != 0:
                    comments_polarity.append(sent_analysis.sentiment.polarity)
            
            for comment in range(len(comment_list)):
                comments['comment_' + str(comment)] = comment_list[comment]                                
                
            for (comment_num, text) in comments.items():
                sent = TextBlob(text)    
                if sent_analysis.sentiment.polarity != 0:
                    comments_pol[comment_num] = sent.sentiment.polarity                    
                
            sent_mean = round(numpy.mean(comments_polarity), 1)        
            app_stats['sentimental_analysis'] = {'mean': sent_mean, 'comment_polarity': comments_pol}    
            
            stats_list.append(app_stats)
            
        return stats_list
{'fraction_non_alphanumeric': 0.11231768338124135, 'fraction_numerical': 0.0061748110294900455, 'mean_word_length': 1.8904615384615384, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6490203', 'n_tokens_mistral': 2826, 'n_tokens_neox': 2744, 'n_words': 491}