<commit_before>from django.core.management.base import BaseCommand, CommandError
from django.contrib.auth.models import User
import json
import datetime
from website.models import UserPageView, Jurisdiction, JurisdictionRating

class Command(BaseCommand):
    args = 'No argument needed'
    help = 'Find the most popular jurisdictions'
    # url names that are for sure not specific jurisdiction
    skip_names = ['browse', 'search', 'autocomplete', 'answer_uploadfile', ]

    def handle(self, *args, **options):
        jurisdiction_unique_users = {}
        
        #find all the UserPageView for the last 7 days
        today = datetime.date.today()
        d = datetime.timedelta(days=-8)
        start_date = today + d
        print 'Updating most popular jurisdictions for days after: ' + str(start_date)
        user_page_views = UserPageView.objects.filter(last_page_view_date__gt=start_date)
        #print 'Number of user page views: ' + str(len(user_page_views))
        
        for user_page_view in user_page_views:
            #print 'Datetime: ' + str(user_page_view.last_page_view_date)
            if user_page_view.url.startswith('/jurisdiction/'):
                name_for_url = user_page_view.url.replace('/jurisdiction/', '')
                name_for_url = name_for_url.partition('/')[0]
                if name_for_url in self.skip_names:
                    continue
                if name_for_url.isdigit():
                    #search for jurisdiction by id
                    jurisdictions = Jurisdiction.objects.filter(id=name_for_url)
                else:
                    #search for jurisdiction by name for url
                    jurisdictions = Jurisdiction.objects.filter(name_for_url=name_for_url)
                if len(jurisdictions) > 0:
                    jurisdiction = jurisdictions[0] #just get 1st
                #print 'name_for_url: ' + name_for_url
                #print 'jurisdiction: ' + jurisdiction.show_jurisdiction()
                
                #remember jurisdiction unique users
                try:
                    jurisdiction_unique_users[jurisdiction.id]
                except:
                    jurisdiction_unique_users[jurisdiction.id] = []
                if user_page_view.user.id not in jurisdiction_unique_users[jurisdiction.id]:
                    jurisdiction_unique_users[jurisdiction.id].append(user_page_view.user.id)
                #print 'jurisdiction_unique_users' + str(jurisdiction_unique_users)
            
        #find the jurisdiction with most unique users
        jurisdiction_unique_users_counts = []
        for jid, user_list in jurisdiction_unique_users.iteritems():
            jurisdiction_unique_users_counts.append([jid, len(user_list)])
        jurisdiction_unique_users_counts.sort(key=lambda x: x[1], reverse=True) #sort by user count
        jurisdiction_unique_users_counts = jurisdiction_unique_users_counts[0:10] #only top 10
        
        #save data in JurisdictionRating
        jurisdiction_ratings = JurisdictionRating.objects.filter(rating_type='V')
        for jurisdiction_rating in jurisdiction_ratings:
            jurisdiction_rating.delete()
        rank = 1
        for jid, count in jurisdiction_unique_users_counts:
            jurisdiction_rating = JurisdictionRating(rating_type='V', rank=rank, value=count)
            jurisdiction = Jurisdiction.objects.get(id=jid)
            jurisdiction_rating.jurisdiction = jurisdiction
            jurisdiction_rating.create_datetime = today
            jurisdiction_rating.save()<commit_msg>Updated existing/ugly 'most_popular' command to use new session tracking<commit_after>from django.core.management.base import BaseCommand, CommandError
from django.contrib.auth.models import User
import json
import datetime
from tracking.models import Pageview, Visitor
from website.models import Jurisdiction

class Command(BaseCommand):
    args = 'No argument needed'
    help = 'Find the most popular jurisdictions'
    # url names that are for sure not specific jurisdiction
    skip_names = ['browse', 'search', 'autocomplete', 'answer_uploadfile', ]

    def handle(self, *args, **options):
        jurisdiction_unique_users = {}
        
        #find all the UserPageView for the last 7 days
        today = datetime.datetime.now()
        d = datetime.timedelta(hours=-24)
        start_date = today + d
        #print 'Updating most popular jurisdictions for days after: ' + str(start_date)
        user_page_views = Pageview.objects.filter(view_time__gt=start_date)
        #print 'Number of user page views: ' + str(len(user_page_views))
        
        for user_page_view in user_page_views:
            #print 'Datetime: ' + str(user_page_view.last_page_view_date)
            if user_page_view.url.startswith('/jurisdiction/'):
                name_for_url = user_page_view.url.replace('/jurisdiction/', '')
                name_for_url = name_for_url.partition('/')[0]
                if name_for_url in self.skip_names:
                    continue
                if name_for_url.isdigit():
                    #search for jurisdiction by id
                    jurisdictions = Jurisdiction.objects.filter(id=name_for_url)
                else:
                    #search for jurisdiction by name for url
                    jurisdictions = Jurisdiction.objects.filter(name_for_url=name_for_url)
                if len(jurisdictions) > 0:
                    jurisdiction = jurisdictions[0] #just get 1st
                #print 'name_for_url: ' + name_for_url
                #print 'jurisdiction: ' + jurisdiction.show_jurisdiction()
                
                #remember jurisdiction unique users
                try:
                    jurisdiction_unique_users[jurisdiction.id]
                except:
                    jurisdiction_unique_users[jurisdiction.id] = []
                if user_page_view.visitor.session_key not in jurisdiction_unique_users[jurisdiction.id]:
                    jurisdiction_unique_users[jurisdiction.id].append(user_page_view.visitor.session_key)
                #print 'jurisdiction_unique_users' + str(jurisdiction_unique_users)
            
        #find the jurisdiction with most unique users
        jurisdiction_unique_users_counts = []
        for jid, user_list in jurisdiction_unique_users.iteritems():
            jurisdiction_unique_users_counts.append([jid, len(user_list)])
        jurisdiction_unique_users_counts.sort(key=lambda x: x[1], reverse=True) #sort by user count
        jurisdiction_unique_users_counts = jurisdiction_unique_users_counts[0:10] #only top 10
        
        #save data in JurisdictionRating
        #jurisdiction_ratings = JurisdictionRating.objects.filter(rating_type='V')
        #for jurisdiction_rating in jurisdiction_ratings:
        #    jurisdiction_rating.delete()
        rank = 1
        for jid, count in jurisdiction_unique_users_counts:
        #    jurisdiction_rating = JurisdictionRating(rating_type='V', rank=rank, value=count)
            jurisdiction = Jurisdiction.objects.get(id=jid)
        #    jurisdiction_rating.jurisdiction = jurisdiction
        #    jurisdiction_rating.create_datetime = today
        #    jurisdiction_rating.save()
{'fraction_non_alphanumeric': 0.07547950876224645, 'fraction_numerical': 0.0037256795915551265, 'mean_word_length': 2.504835589941973, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5315082', 'n_tokens_mistral': 2071, 'n_tokens_neox': 1890, 'n_words': 428}