<commit_before>import requests
import json
import os
import datetime as dt
import dateutil.parser as time
import logging
import re
from uuid import uuid4
from xml.etree import ElementTree
from html_unescaping import unescape
# see https://github.com/jdepoix/youtube-transcript-api/blob/master/src/transcript_api.py
data_dir = 'data/'

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

# Console handler
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.DEBUG)
# format
formatter = logging.Formatter('%(filename)s ## [%(asctime)s] -- %(levelname)s == "%(message)s"')
stream_handler.setFormatter(formatter)
# add handler
logger.addHandler(stream_handler)

# FILE HANDLER LOGGER (not implemented again)
# from logging.handlers import RotatingFileHandler
# # création d'un handler qui va rediriger une écriture du log vers
# # un fichier en mode 'append', avec 1 backup et une taille max de 1Mo
# file_handler = RotatingFileHandler('activity.log', 'a', 1000000, 1)
# # on lui met le niveau sur DEBUG, on lui dit qu'il doit utiliser le formateur
# # créé précédement et on ajoute ce handler au logger
# file_handler.setLevel(logging.DEBUG)
# # création d'un formateur qui va ajouter le temps, le niveau
# # de chaque message quand on écrira un message dans le log
# formatter = logging.Formatter('%(asctime)s :: %(levelname)s :: %(message)s')
# file_handler.setFormatter(formatter)
# logger.addHandler(file_handler)


##########################################################################
# Youtube Data Api request
##########################################################################
class YouTube():
    #api_key = None
    #access_token = None
    #api_base_url = 'https://www.googleapis.com/youtube/v3/'
    #part = None

    def __init__(self, api_key, access_token=None, part=None,api_url=None):
        self.api_key = api_key
        self.access_token = access_token
        self.api_base_url = 'https://www.googleapis.com/youtube/v3/'
        if part:
            self.part = part
        if api_url:
            self.api_url = api_url

    # make req
    def try_request(self, kwargs, endpoint):
        url = self.api_base_url + endpoint
        try:
            req = requests.get(url, kwargs)
            logger.info('try_request success on ' +  url)
            logger.debug('kwargs are ' + str(kwargs))
        except requests.exceptions.RequestException as e:
            logger.warning('try_request failed on ' + url)
            logger.warning('error is : ' + e)
        return self.response(req)

    # prepare request with same obligatory param
    def get_query(self, endpoint, **kwargs):
        if self.access_token:
            kwargs['access_token'] = self.access_token
        else:
            kwargs['key'] = self.api_key
        if 'part' not in kwargs:
            kwargs['part'] = self.part
        kwargs = json.dumps(kwargs)
        kwargs = json.loads(kwargs)
        return self.try_request(kwargs, endpoint)

    def get_search(api_key, session):
        search_results = YouTube(api_key).get_query(
            'search',
            q=session['q'],
            part=session['part'],
            relevanceLanguage=session['language'],
            maxResults=session['maxResults'],
            ranking=session['ranking']
        )
        return search_results

    def get_channel(api_key, session):
        channel_results = YouTube(api_key).get_query(
            'search',
            channelId=session['channelId'],
            part=session['part'],
            maxResults=session['maxResults']
        )
        return channel_results

    # TODO
    def get_playlist(api_key, session):
        playlist_results = YouTube(api_key).get_query(
            'playlistItems',
            playlistId=session['playlistId'],
            part=session['part'],
            maxResults=session['maxResults']
        )
        return playlist_results

    def verify_error(api_key, result_query):
        if not 'error' in result_query:
            return result_query
        else:
            logger.errors('Reason of error is : ' + commentThread['error']['errors'][0]['reason'])
            return commentThread['error']

    @staticmethod
    def response(response):
        return response.json()

##########################################################################
# Video
# rename in aggregate class (because of list of videos)
##########################################################################
class Videos():
    # to complete
    video_fields = ['id','videoId','query_id','kind']
    snippet_fields = ['']
    stats_fields = ['']

    def __init__(self, mongo_curs, id=None, query_id=None):
        self.db = mongo_curs.db
        if id:
            self.id_video = id
            self.get()
        if query_id:
            self.query_id = query_id
            self.get()

    def get_one_video(self):
        try:
            current_video = self.db.videos.find_one_or_404({ 'id_video': self.id_video})
            logger.debug('get video : ', current_video['id_video'])
        except BaseException as e:
            logger.error('video not found : ', e)
        return

    def get_one_query_videos(self):
        try:
            current_videos = self.db.videos.find({ 'query_id': self.query_id})
            logger.debug('get one query videos : ', current_user['query_id'])
        except BaseException as e:
            logger.error('one query videos not found : ', e)
        return

    def view_one_video(self):
        return str(current_video)

    def view_query_videos(self):
        return str(current_video)    

    # def update(self, dataUser):
    #     user_update = self.db.users.update_one(
    #       { 'id_pytheas' : self.id_pytheas },
    #       { '$set': { 'username': dataUser['username']} }
    #     )
    #     return user_update

    # def add_metrics_for_each(self, list_video):
    #     for each in list_video:
    #         print(each)
    #         # if 'youtube#video' in result['kind']:
    #             # print(results['kind'])
    #     return

    def delete():
        return


##########################################################################
# Comment
# rename in aggregate class (because of list of videos)
##########################################################################
class Comment():
    def __init__(self, mongo_curs, query_id):
        #super(Executive, self).__init__(*args)
        self.db = mongo_curs.db
        self.query_id = query_id

    def create_if_not_exist(self, video_id):
        query_id = self.query_id
        try:
            current_caption = self.db.captions.find_one_or_404(
                { '$and':[{ 'query_id': query_id }, { 'videoId': video_id }] }
            )
        except BaseException as e:
            self.create_captions(id_video)
            logger.warning(
                str('comment not found or error. Log is here : ') + str(e) + str(type(e))
            )
        return

    def create_comment_for_each(self, commentThread):
        if not 'error' in commentThread:
            nb_comment = 0
            for each in commentThread['items']:
                nb_comment += 1
                
                snippet = each['snippet']['topLevelComment']['snippet']
                if 'authorChannelId' in snippet:
                    snippet['authorChannelId'] = snippet['authorChannelId']['value']
                else:
                    snippet['authorChannelId'] = 'cortext_pytheas_unknown_author#' + each['id']
                
                topCommentIntegrated = {
                    'id' : each['id'],
                    'query_id' : self.query_id,
                    'isPublic': each['snippet']['isPublic'],
                    'canReply': each['snippet']['canReply'],
                    'totalReplyCount': each['snippet']['totalReplyCount'],
                    'videoId' : each['snippet']['videoId'],
                    'snippet': snippet,
                    'is_top_level_comment': 'true',
                }
                self.db.comments.insert_one(topCommentIntegrated)
                
                if 'replies' in each:
                    for child in each['replies']['comments']:
                        nb_comment += 1
                        snippet = child['snippet']
                        if 'authorChannelId' in snippet:
                            snippet['authorChannelId'] = snippet['authorChannelId']['value']
                        else:
                            snippet['authorChannelId'] = 'cortext_pytheas_unknown_author#' + each['id']

                        CommentIntegrated = {
                            'id' : child['id'],
                            'query_id' : self.query_id,
                            'videoId' : child['snippet']['videoId'],
                            'snippet': snippet,
                            'is_top_level_comment': 'false'
                        }
                        self.db.comments.insert_one(CommentIntegrated)
            logger.debug('total nb comment for this request = ' + str(nb_comment))

        else:
            logger.error(commentThread['error'])
            logger.error('reason of error is : ' + ['errors'][0]['reason'])
            return

        return

    # def add_stats_for_each(self, list_video):<commit_msg>minor bug on logging errors comments<commit_after>import requests
import json
import os
import datetime as dt
import dateutil.parser as time
import logging
import re
from uuid import uuid4
from xml.etree import ElementTree
from html_unescaping import unescape
# see https://github.com/jdepoix/youtube-transcript-api/blob/master/src/transcript_api.py
data_dir = 'data/'

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

# Console handler
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.DEBUG)
# format
formatter = logging.Formatter('%(filename)s ## [%(asctime)s] -- %(levelname)s == "%(message)s"')
stream_handler.setFormatter(formatter)
# add handler
logger.addHandler(stream_handler)

# FILE HANDLER LOGGER (not implemented again)
# from logging.handlers import RotatingFileHandler
# # création d'un handler qui va rediriger une écriture du log vers
# # un fichier en mode 'append', avec 1 backup et une taille max de 1Mo
# file_handler = RotatingFileHandler('activity.log', 'a', 1000000, 1)
# # on lui met le niveau sur DEBUG, on lui dit qu'il doit utiliser le formateur
# # créé précédement et on ajoute ce handler au logger
# file_handler.setLevel(logging.DEBUG)
# # création d'un formateur qui va ajouter le temps, le niveau
# # de chaque message quand on écrira un message dans le log
# formatter = logging.Formatter('%(asctime)s :: %(levelname)s :: %(message)s')
# file_handler.setFormatter(formatter)
# logger.addHandler(file_handler)


##########################################################################
# Youtube Data Api request
##########################################################################
class YouTube():
    #api_key = None
    #access_token = None
    #api_base_url = 'https://www.googleapis.com/youtube/v3/'
    #part = None

    def __init__(self, api_key, access_token=None, part=None,api_url=None):
        self.api_key = api_key
        self.access_token = access_token
        self.api_base_url = 'https://www.googleapis.com/youtube/v3/'
        if part:
            self.part = part
        if api_url:
            self.api_url = api_url

    # make req
    def try_request(self, kwargs, endpoint):
        url = self.api_base_url + endpoint
        try:
            req = requests.get(url, kwargs)
            logger.info('try_request success on ' +  url)
            logger.debug('kwargs are ' + str(kwargs))
        except requests.exceptions.RequestException as e:
            logger.warning('try_request failed on ' + url)
            logger.warning('error is : ' + e)
        return self.response(req)

    # prepare request with same obligatory param
    def get_query(self, endpoint, **kwargs):
        if self.access_token:
            kwargs['access_token'] = self.access_token
        else:
            kwargs['key'] = self.api_key
        if 'part' not in kwargs:
            kwargs['part'] = self.part
        kwargs = json.dumps(kwargs)
        kwargs = json.loads(kwargs)
        return self.try_request(kwargs, endpoint)

    def get_search(api_key, session):
        search_results = YouTube(api_key).get_query(
            'search',
            q=session['q'],
            part=session['part'],
            relevanceLanguage=session['language'],
            maxResults=session['maxResults'],
            ranking=session['ranking']
        )
        return search_results

    def get_channel(api_key, session):
        channel_results = YouTube(api_key).get_query(
            'search',
            channelId=session['channelId'],
            part=session['part'],
            maxResults=session['maxResults']
        )
        return channel_results

    # TODO
    def get_playlist(api_key, session):
        playlist_results = YouTube(api_key).get_query(
            'playlistItems',
            playlistId=session['playlistId'],
            part=session['part'],
            maxResults=session['maxResults']
        )
        return playlist_results

    def verify_error(api_key, result_query):
        if not 'error' in result_query:
            return result_query
        else:
            logger.errors('Reason of error is : ' + commentThread['error']['errors'][0]['reason'])
            return commentThread['error']

    @staticmethod
    def response(response):
        return response.json()

##########################################################################
# Video
# rename in aggregate class (because of list of videos)
##########################################################################
class Videos():
    # to complete
    video_fields = ['id','videoId','query_id','kind']
    snippet_fields = ['']
    stats_fields = ['']

    def __init__(self, mongo_curs, id=None, query_id=None):
        self.db = mongo_curs.db
        if id:
            self.id_video = id
            self.get()
        if query_id:
            self.query_id = query_id
            self.get()

    def get_one_video(self):
        try:
            current_video = self.db.videos.find_one_or_404({ 'id_video': self.id_video})
            logger.debug('get video : ', current_video['id_video'])
        except BaseException as e:
            logger.error('video not found : ', e)
        return

    def get_one_query_videos(self):
        try:
            current_videos = self.db.videos.find({ 'query_id': self.query_id})
            logger.debug('get one query videos : ', current_user['query_id'])
        except BaseException as e:
            logger.error('one query videos not found : ', e)
        return

    def view_one_video(self):
        return str(current_video)

    def view_query_videos(self):
        return str(current_video)    

    # def update(self, dataUser):
    #     user_update = self.db.users.update_one(
    #       { 'id_pytheas' : self.id_pytheas },
    #       { '$set': { 'username': dataUser['username']} }
    #     )
    #     return user_update

    # def add_metrics_for_each(self, list_video):
    #     for each in list_video:
    #         print(each)
    #         # if 'youtube#video' in result['kind']:
    #             # print(results['kind'])
    #     return

    def delete():
        return


##########################################################################
# Comment
# rename in aggregate class (because of list of videos)
##########################################################################
class Comment():
    def __init__(self, mongo_curs, query_id):
        #super(Executive, self).__init__(*args)
        self.db = mongo_curs.db
        self.query_id = query_id

    def create_if_not_exist(self, video_id):
        query_id = self.query_id
        try:
            current_caption = self.db.captions.find_one_or_404(
                { '$and':[{ 'query_id': query_id }, { 'videoId': video_id }] }
            )
        except BaseException as e:
            self.create_captions(id_video)
            logger.warning(
                str('comment not found or error. Log is here : ') + str(e) + str(type(e))
            )
        return

    def create_comment_for_each(self, commentThread):
        if not 'error' in commentThread:
            nb_comment = 0
            for each in commentThread['items']:
                nb_comment += 1
                
                snippet = each['snippet']['topLevelComment']['snippet']
                if 'authorChannelId' in snippet:
                    snippet['authorChannelId'] = snippet['authorChannelId']['value']
                else:
                    snippet['authorChannelId'] = 'cortext_pytheas_unknown_author#' + each['id']
                
                topCommentIntegrated = {
                    'id' : each['id'],
                    'query_id' : self.query_id,
                    'isPublic': each['snippet']['isPublic'],
                    'canReply': each['snippet']['canReply'],
                    'totalReplyCount': each['snippet']['totalReplyCount'],
                    'videoId' : each['snippet']['videoId'],
                    'snippet': snippet,
                    'is_top_level_comment': 'true',
                }
                self.db.comments.insert_one(topCommentIntegrated)
                
                if 'replies' in each:
                    for child in each['replies']['comments']:
                        nb_comment += 1
                        snippet = child['snippet']
                        if 'authorChannelId' in snippet:
                            snippet['authorChannelId'] = snippet['authorChannelId']['value']
                        else:
                            snippet['authorChannelId'] = 'cortext_pytheas_unknown_author#' + each['id']

                        CommentIntegrated = {
                            'id' : child['id'],
                            'query_id' : self.query_id,
                            'videoId' : child['snippet']['videoId'],
                            'snippet': snippet,
                            'is_top_level_comment': 'false'
                        }
                        self.db.comments.insert_one(CommentIntegrated)
            logger.debug('total nb comment for this request = ' + str(nb_comment))

        else:
            logger.error(commentThread['error'])
            logger.error('reason of error is : ' + commentThread['error']['errors'][0]['reason'])

        return

    # def add_stats_for_each(self, list_video):
{'fraction_non_alphanumeric': 0.15291729001923487, 'fraction_numerical': 0.0025646505663603335, 'mean_word_length': 2.2579634464751956, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3245450', 'n_tokens_mistral': 5220, 'n_tokens_neox': 5020, 'n_words': 1133}