<commit_before>import datetime
import re
import random
import time
import redis
from utils import log as logging
from django.http import HttpResponse
from django.conf import settings
from django.db import connection
from django.template import Template, Context
from apps.statistics.rstats import round_time
from utils import json_functions as json

class LastSeenMiddleware(object):
    def __init__(self, get_response=None):
        self.get_response = get_response

    def process_response(self, request, response):
        if ((request.path == '/' or
             request.path.startswith('/reader/refresh_feeds') or
             request.path.startswith('/reader/load_feeds') or
             request.path.startswith('/reader/feeds'))
            and hasattr(request, 'user')
            and request.user.is_authenticated): 
            hour_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes=60)
            ip = request.META.get('HTTP_X_FORWARDED_FOR', None) or request.META['REMOTE_ADDR']
            # SUBSCRIBER_EXPIRE = datetime.datetime.utcnow() - datetime.timedelta(days=settings.SUBSCRIBER_EXPIRE)
            if request.user.profile.last_seen_on < hour_ago:
                logging.user(request, "~FG~BBRepeat visitor: ~SB%s (%s)" % (
                    request.user.profile.last_seen_on, ip))
                from apps.profile.tasks import CleanupUser
                CleanupUser.delay(user_id=request.user.pk)
            elif settings.DEBUG:
                logging.user(request, "~FG~BBRepeat visitor (ignored): ~SB%s (%s)" % (
                    request.user.profile.last_seen_on, ip))

            request.user.profile.last_seen_on = datetime.datetime.utcnow()
            request.user.profile.last_seen_ip = ip[-15:]
            request.user.profile.save()
        
        return response

    def __call__(self, request):
        response = None
        if hasattr(self, 'process_request'):
            response = self.process_request(request)
        if not response:
            response = self.get_response(request)
        if hasattr(self, 'process_response'):
            response = self.process_response(request, response)

        return response

class DBProfilerMiddleware:
    def __init__(self, get_response=None):
        self.get_response = get_response

    def process_request(self, request): 
        setattr(request, 'activated_segments', [])
        if ((request.path.startswith('/reader/feed') or
             request.path.startswith('/reader/river')) and
            random.random() < .01):
            request.activated_segments.append('db_profiler')
            connection.use_debug_cursor = True
            setattr(settings, 'ORIGINAL_DEBUG', settings.DEBUG)
            settings.DEBUG = True

    def process_celery(self): 
        setattr(self, 'activated_segments', [])        
        if random.random() < .01:
            self.activated_segments.append('db_profiler')
            connection.use_debug_cursor = True
            setattr(settings, 'ORIGINAL_DEBUG', settings.DEBUG)
            settings.DEBUG = True
            return self
    
    def process_exception(self, request, exception):
        if hasattr(request, 'sql_times_elapsed'):
            self._save_times(request.sql_times_elapsed)

    def process_response(self, request, response):
        if hasattr(request, 'sql_times_elapsed'):
            # middleware = SQLLogToConsoleMiddleware()
            # middleware.process_celery(self)
            # logging.debug(" ---> ~FGProfiling~FB app: %s" % request.sql_times_elapsed)
            self._save_times(request.sql_times_elapsed)
        return response
    
    def process_celery_finished(self):
        middleware = SQLLogToConsoleMiddleware()
        middleware.process_celery(self)
        if hasattr(self, 'sql_times_elapsed'):
            logging.debug(" ---> ~FGProfiling~FB task: %s" % self.sql_times_elapsed)
            self._save_times(self.sql_times_elapsed, 'task_')
    
    def process_request_finished(self):
        middleware = SQLLogToConsoleMiddleware()
        middleware.process_celery(self)
        if hasattr(self, 'sql_times_elapsed'):
            logging.debug(" ---> ~FGProfiling~FB app: %s" % self.sql_times_elapsed)
            self._save_times(self.sql_times_elapsed, 'app_')
    
    def _save_times(self, db_times, prefix=""):
        if not db_times: return
        
        r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL)
        pipe = r.pipeline()
        minute = round_time(round_to=60)
        for db, duration in list(db_times.items()):
            key = "DB:%s%s:%s" % (prefix, db, minute.strftime('%s'))
            pipe.incr("%s:c" % key)
            pipe.expireat("%s:c" % key, (minute + datetime.timedelta(days=2)).strftime("%s"))
            if duration:
                pipe.incrbyfloat("%s:t" % key, duration)
                pipe.expireat("%s:t" % key, (minute + datetime.timedelta(days=2)).strftime("%s"))
        pipe.execute()

    def __call__(self, request):
        response = None
        if hasattr(self, 'process_request'):
            response = self.process_request(request)
        if not response:
            response = self.get_response(request)
        if hasattr(self, 'process_response'):
            response = self.process_response(request, response)

        return response

class SQLLogToConsoleMiddleware:
    def __init__(self, get_response=None):
        self.get_response = get_response

    def activated(self, request):
        return (settings.DEBUG_QUERIES or 
                (hasattr(request, 'activated_segments') and
                 'db_profiler' in request.activated_segments))

    def process_response(self, request, response): 
        if not self.activated(request): return response
        if connection.queries:
            time_elapsed = sum([float(q['time']) for q in connection.queries])
            queries = connection.queries
            if getattr(connection, 'queriesx', False):
                queries.extend(connection.queriesx)
                connection.queriesx = []
            for query in queries:
                if query.get('mongo'):
                    query['sql'] = "~FM%s: %s" % (query['mongo']['collection'], query['mongo']['query'])
                elif query.get('db_redis'):
                    query['sql'] = "~FC%s" % (query['db_redis']['query'])
                elif query.get('redis'):
                    query['sql'] = "~FC%s" % (query['redis']['query'])
                elif query.get('redis_user'):
                    query['sql'] = "~FC%s" % (query['redis_user']['query'])
                elif query.get('redis_story'):
                    query['sql'] = "~FC%s" % (query['redis_story']['query'])
                elif query.get('redis_session'):
                    query['sql'] = "~FC%s" % (query['redis_session']['query'])
                else:
                    query['sql'] = re.sub(r'SELECT (.*?) FROM', 'SELECT * FROM', query['sql'])
                    query['sql'] = re.sub(r'SELECT', '~FYSELECT', query['sql'])
                    query['sql'] = re.sub(r'INSERT', '~FGINSERT', query['sql'])
                    query['sql'] = re.sub(r'UPDATE', '~FY~SBUPDATE', query['sql'])
                    query['sql'] = re.sub(r'DELETE', '~FR~SBDELETE', query['sql'])
            if settings.DEBUG and settings.DEBUG_QUERIES and not getattr(settings, 'DEBUG_QUERIES_SUMMARY_ONLY', False):
                t = Template("{% for sql in sqllog %}{% if not forloop.first %}                  {% endif %}[{{forloop.counter}}] ~FC{{sql.time}}s~FW: {{sql.sql|safe}}{% if not forloop.last %}\n{% endif %}{% endfor %}")
                logging.debug(t.render(Context({
                    'sqllog': queries,
                    'count': len(queries),
                    'time': time_elapsed,
                })))
            times_elapsed = {
                'sql': sum([float(q['time']) 
                           for q in queries if not q.get('mongo') and 
                                               not q.get('redis_user') and
                                               not q.get('redis_story') and
                                               not q.get('redis_session') and
                                               not q.get('db_redis')]),
                'mongo': sum([float(q['time']) for q in queries if q.get('mongo')]),
                'db_redis': sum([float(q['time']) for q in queries if q.get('db_redis')]),
                'redis_user': sum([float(q['time']) for q in queries if q.get('redis_user')]),<commit_msg>Log missing queries<commit_after>import datetime
import re
import random
import time
import redis
from utils import log as logging
from django.http import HttpResponse
from django.conf import settings
from django.db import connection
from django.template import Template, Context
from apps.statistics.rstats import round_time
from utils import json_functions as json

class LastSeenMiddleware(object):
    def __init__(self, get_response=None):
        self.get_response = get_response

    def process_response(self, request, response):
        if ((request.path == '/' or
             request.path.startswith('/reader/refresh_feeds') or
             request.path.startswith('/reader/load_feeds') or
             request.path.startswith('/reader/feeds'))
            and hasattr(request, 'user')
            and request.user.is_authenticated): 
            hour_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes=60)
            ip = request.META.get('HTTP_X_FORWARDED_FOR', None) or request.META['REMOTE_ADDR']
            # SUBSCRIBER_EXPIRE = datetime.datetime.utcnow() - datetime.timedelta(days=settings.SUBSCRIBER_EXPIRE)
            if request.user.profile.last_seen_on < hour_ago:
                logging.user(request, "~FG~BBRepeat visitor: ~SB%s (%s)" % (
                    request.user.profile.last_seen_on, ip))
                from apps.profile.tasks import CleanupUser
                CleanupUser.delay(user_id=request.user.pk)
            elif settings.DEBUG:
                logging.user(request, "~FG~BBRepeat visitor (ignored): ~SB%s (%s)" % (
                    request.user.profile.last_seen_on, ip))

            request.user.profile.last_seen_on = datetime.datetime.utcnow()
            request.user.profile.last_seen_ip = ip[-15:]
            request.user.profile.save()
        
        return response

    def __call__(self, request):
        response = None
        if hasattr(self, 'process_request'):
            response = self.process_request(request)
        if not response:
            response = self.get_response(request)
        if hasattr(self, 'process_response'):
            response = self.process_response(request, response)

        return response

class DBProfilerMiddleware:
    def __init__(self, get_response=None):
        self.get_response = get_response

    def process_request(self, request): 
        setattr(request, 'activated_segments', [])
        if ((request.path.startswith('/reader/feed') or
             request.path.startswith('/reader/river')) and
            random.random() < .01):
            request.activated_segments.append('db_profiler')
            connection.use_debug_cursor = True
            setattr(settings, 'ORIGINAL_DEBUG', settings.DEBUG)
            settings.DEBUG = True

    def process_celery(self): 
        setattr(self, 'activated_segments', [])        
        if random.random() < .01:
            self.activated_segments.append('db_profiler')
            connection.use_debug_cursor = True
            setattr(settings, 'ORIGINAL_DEBUG', settings.DEBUG)
            settings.DEBUG = True
            return self
    
    def process_exception(self, request, exception):
        if hasattr(request, 'sql_times_elapsed'):
            self._save_times(request.sql_times_elapsed)

    def process_response(self, request, response):
        if hasattr(request, 'sql_times_elapsed'):
            # middleware = SQLLogToConsoleMiddleware()
            # middleware.process_celery(self)
            # logging.debug(" ---> ~FGProfiling~FB app: %s" % request.sql_times_elapsed)
            self._save_times(request.sql_times_elapsed)
        return response
    
    def process_celery_finished(self):
        middleware = SQLLogToConsoleMiddleware()
        middleware.process_celery(self)
        if hasattr(self, 'sql_times_elapsed'):
            logging.debug(" ---> ~FGProfiling~FB task: %s" % self.sql_times_elapsed)
            self._save_times(self.sql_times_elapsed, 'task_')
    
    def process_request_finished(self):
        middleware = SQLLogToConsoleMiddleware()
        middleware.process_celery(self)
        if hasattr(self, 'sql_times_elapsed'):
            logging.debug(" ---> ~FGProfiling~FB app: %s" % self.sql_times_elapsed)
            self._save_times(self.sql_times_elapsed, 'app_')
    
    def _save_times(self, db_times, prefix=""):
        if not db_times: return
        
        r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL)
        pipe = r.pipeline()
        minute = round_time(round_to=60)
        for db, duration in list(db_times.items()):
            key = "DB:%s%s:%s" % (prefix, db, minute.strftime('%s'))
            pipe.incr("%s:c" % key)
            pipe.expireat("%s:c" % key, (minute + datetime.timedelta(days=2)).strftime("%s"))
            if duration:
                pipe.incrbyfloat("%s:t" % key, duration)
                pipe.expireat("%s:t" % key, (minute + datetime.timedelta(days=2)).strftime("%s"))
        pipe.execute()

    def __call__(self, request):
        response = None
        if hasattr(self, 'process_request'):
            response = self.process_request(request)
        if not response:
            response = self.get_response(request)
        if hasattr(self, 'process_response'):
            response = self.process_response(request, response)

        return response

class SQLLogToConsoleMiddleware:
    def __init__(self, get_response=None):
        self.get_response = get_response

    def activated(self, request):
        return (settings.DEBUG_QUERIES or 
                (hasattr(request, 'activated_segments') and
                 'db_profiler' in request.activated_segments))

    def process_response(self, request, response): 
        if not self.activated(request): return response
        if connection.queries:
            time_elapsed = sum([float(q['time']) for q in connection.queries])
            queries = connection.queries
            if getattr(connection, 'queriesx', False):
                queries.extend(connection.queriesx)
                connection.queriesx = []
            for query in queries:
                if query.get('mongo'):
                    query['sql'] = "~FM%s: %s" % (query['mongo']['collection'], query['mongo']['query'])
                elif query.get('db_redis'):
                    query['sql'] = "~FC%s" % (query['db_redis']['query'])
                elif query.get('redis'):
                    query['sql'] = "~FC%s" % (query['redis']['query'])
                elif query.get('redis_user'):
                    query['sql'] = "~FC%s" % (query['redis_user']['query'])
                elif query.get('redis_story'):
                    query['sql'] = "~FC%s" % (query['redis_story']['query'])
                elif query.get('redis_session'):
                    query['sql'] = "~FC%s" % (query['redis_session']['query'])
                elif 'sql' not in query:
                    logging.debug(" ***> Query log missing: %s" % query)
                else:
                    query['sql'] = re.sub(r'SELECT (.*?) FROM', 'SELECT * FROM', query['sql'])
                    query['sql'] = re.sub(r'SELECT', '~FYSELECT', query['sql'])
                    query['sql'] = re.sub(r'INSERT', '~FGINSERT', query['sql'])
                    query['sql'] = re.sub(r'UPDATE', '~FY~SBUPDATE', query['sql'])
                    query['sql'] = re.sub(r'DELETE', '~FR~SBDELETE', query['sql'])
            if settings.DEBUG and settings.DEBUG_QUERIES and not getattr(settings, 'DEBUG_QUERIES_SUMMARY_ONLY', False):
                t = Template("{% for sql in sqllog %}{% if not forloop.first %}                  {% endif %}[{{forloop.counter}}] ~FC{{sql.time}}s~FW: {{sql.sql|safe}}{% if not forloop.last %}\n{% endif %}{% endfor %}")
                logging.debug(t.render(Context({
                    'sqllog': queries,
                    'count': len(queries),
                    'time': time_elapsed,
                })))
            times_elapsed = {
                'sql': sum([float(q['time']) 
                           for q in queries if not q.get('mongo') and 
                                               not q.get('redis_user') and
                                               not q.get('redis_story') and
                                               not q.get('redis_session') and
                                               not q.get('db_redis')]),
                'mongo': sum([float(q['time']) for q in queries if q.get('mongo')]),
                'db_redis': sum([float(q['time']) for q in queries if q.get('db_redis')]),
                'redis_user': sum([float(q['time']) for q in queries if q.get('redis_user')]),
{'fraction_non_alphanumeric': 0.12879842263975877, 'fraction_numerical': 0.0013917884481558804, 'mean_word_length': 2.1706195991910278, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3070267', 'n_tokens_mistral': 5031, 'n_tokens_neox': 4858, 'n_words': 972}