<commit_before>#!/usr/bin/python
#
# File:   main.py
# Date:   25-Oct-14
# Author: <NAME> <<EMAIL>>
#
# Analytics dashboard for courses running on edx-platform.
# 
# Top-level module.

import logging
import os

import re
import json
import webapp2
import datetime

import gsdata
import bqutil
import auth
import local_config
import urllib

from unidecode import unidecode

from auth import auth_required
from stats import DataStats
from datatable import DataTableField
from datasource import DataSource
from dashboard import DashboardRoutes
from collections import defaultdict, OrderedDict

import jinja2

# from gviz_data_table import encode
# from gviz_data_table import Table

from google.appengine.api import memcache

mem = memcache.Client()

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
    extensions=['jinja2.ext.autoescape'],
    autoescape=True)

#-----------------------------------------------------------------------------

class MainPage(auth.AuthenticatedHandler, DataStats, DataSource):
    '''
    Main python class which displays views.
    '''

    @auth_required
    def get_main(self):
        '''
        Main page: show list of all courses
        '''
        courses = self.get_course_listings()

        # logging.info('course listings: %s' % courses['data'])

        html = self.list2table(map(DataTableField, 
                                   [{'field': 'launch', 'title':'Course launch', 'width': '12%'},
                                    {'field': 'course_number', 'title': 'Course #', 'width': '5%'}, 
                                    {'field': 'course_image', 'title': 'Course image'}, 
                                    {'field': 'title', 'title': 'Course Title', 'width': '40%'},
                                    {'field': 'course_id', 'title': 'course ID', 'width': '12%'},
                                   ]), 
                               courses['data'])

        data = self.common_data
        data.update({'data': {},
                     'is_staff': self.is_superuser(),
                     'table': html,
                 })
        template = JINJA_ENVIRONMENT.get_template('courses.html')
        self.response.out.write(template.render(data))

    @auth_required
    def get_admin(self):
        '''
        Admin page: show authorized users, clear cache
        '''
        if not self.user in self.AUTHORIZED_USERS:	# require superuser
            return self.no_auth_sorry()

        msg = ""
        action = self.request.POST.get('action', None)

        if action=='Flush cache':
            memcache.flush_all()
            msg = "Cache flushed"

        elif action=='Reload staff table':
            self.get_staff_table(reload=True)
            msg = "Staff table reloaded - please refresh this page"

        elif action=='Add staff':
            fields = ['username', 'role', 'course_id', 'notes']
            data = { x: (self.request.POST.get(x) or '') for x in fields }
            self.add_staff_table_entry(data)
            msg = "New staff %s added" % data

        todelete = self.request.POST.get('do-delete', None)
        if todelete is not None:
            self.disable_staff_table_entry(int(todelete))
            msg = "Deleted staff table row %s" % todelete

        stable = self.get_staff_table()

        stafftable = self.list2table([DataTableField({'icon':'delete', 'field': 'sid', 'title':' '}), 
                                      'username', 'role', 'course_id', 'notes'], stable)

        data = self.common_data
        data.update({'superusers': self.AUTHORIZED_USERS,
                     'table': stafftable,
                     'msg': msg,
                 })
        template = JINJA_ENVIRONMENT.get_template('admin.html')
        self.response.out.write(template.render(data))

    @auth_required
    def ajax_get_usage_stats(self, org=None, number=None, semester=None):

        course_id = '/'.join([org, number, semester])
        usdat = self.compute_usage_stats(course_id)['data'][0]
        # categories = ['registered', 'viewed', 'explored']
        data = {'data': [usdat] }

        # logging.info('ajax_get_usage_stats data=%s' % data)

        self.response.headers['Content-Type'] = 'application/json'   
        self.response.out.write(json.dumps(data))
        
    @auth_required
    def ajax_get_enrollment_stats(self, org=None, number=None, semester=None):
        '''
        Return enrollment stats in "series" format for HighCharts.
        Use course listings to determine start date
        '''
        course_id = '/'.join([org, number, semester])
        courses = self.get_course_listings()
        start = courses['data_by_key'][course_id]['launch']

        (y,m,d) = map(int, start.split('-'))
        start_dt = datetime.datetime(y, m, d)
        start_dt = start_dt - datetime.timedelta(days=32*9)	# start plot 9 months before launch
        start_str = start_dt.strftime('%Y-%m-%d')

        end_dt = start_dt + datetime.timedelta(days=32*1)	# default position for end selector
        end_str = end_dt.strftime('%Y-%m-%d')
        logging.info('initial start_dt=%s, end_dt=%s' % (start_dt, end_dt))

        bqdat = self.compute_enrollment_by_day(course_id, start=start_str)
        dtrange = ['2199-12-31', '1900-01-01']

        def getrow(x, field, scale):
            dtstr = x['date']
            if dtstr < dtrange[0]:	# min max dates from actual data
                dtrange[0] = dtstr
            if dtstr > dtrange[1]:
                dtrange[1] = dtstr
            ts = self.datetime2milliseconds(dtstr=dtstr)  # (dt - datetime.datetime(1970, 1, 1)).total_seconds() * 1000
            cnt = int(x[field])
            return [ts, cnt]

        def getseries(field):
            return {'name': field['name'], 'data': [ getrow(x, field['field'], field['scale']) for x in bqdat['data'] ]}

        def mkf(field, name, scale=1):
            return {'field': field, 'name': name, 'scale': scale}

        fields = [mkf('nenroll_cum', 'cumulative enrollment')]
        stats = [ getseries(sname) for sname in fields ]

        dtrange = [ datetime.datetime(*map(int, x.split('-'))) for x in dtrange ]
        start_dt = max(dtrange[0], start_dt)
        end_dt =  max(dtrange[0], end_dt)
        if end_dt == start_dt:
            end_dt = end_dt + datetime.timedelta(days=32*2)
        end_dt =  min(dtrange[1], end_dt)
        logging.info('dtrange=%s, actual start_dt=%s, end_dt=%s' % (dtrange, start_dt, end_dt))

        data = {'series': stats,
                'start_dt': self.datetime2milliseconds(start_dt),
                'end_dt': self.datetime2milliseconds(end_dt),
        }

        self.response.headers['Content-Type'] = 'application/json'   
        self.response.out.write(json.dumps(data))
        
    def setup_module_info(self, org=None, number=None, semester=None, url_name=None):
        '''
        setup common module info for problem, video, html
        '''
        course_id = '/'.join([org, number, semester])
        caxis = self.load_course_axis(course_id)

        # get problem info
        the_module = caxis[url_name]
        module_id = the_module['module_id']
        chapter_mid = the_module['chapter_mid']
        chapter_url_name = chapter_mid.rsplit('/',1)[-1]
        name = the_module['name']

        data = self.common_data
        data.update({'course_id': course_id,
                     'name': name,
                     'chapter_mid': chapter_mid,
                     'cun': chapter_url_name,
                     'base': self.get_base(course_id),
                     'url_name': url_name,
                     'module_id': module_id,
                     'module': the_module,
                     'caxis': caxis,
                 })
        return data

    @auth_required
    def get_problem(self, **kwargs):
        '''
        single problem analytics view

        - iframe
        '''

        data = self.setup_module_info(**kwargs)
        # template = os.path.join(os.path.dirname(__file__), 'problem.html')
        template = JINJA_ENVIRONMENT.get_template('problem.html')
        self.response.out.write(template.render(data))


    @auth_required
    def ajax_get_problem_stats(self, org=None, number=None, semester=None, problem_url_name=None):

        course_id = '/'.join([org, number, semester])

        ps = self.compute_problem_stats(course_id)
        # logging.info('problem url_name = %s' % problem_url_name)
        pstats = ps['data_by_key'][problem_url_name]
        # logging.info('pstats = %s' % pstats)

        data = {'data': [ pstats ] }

        self.response.headers['Content-Type'] = 'application/json'   <commit_msg>if unknown problem_url_name retrieved in problem stats don't fail<commit_after>#!/usr/bin/python
#
# File:   main.py
# Date:   25-Oct-14
# Author: <NAME> <<EMAIL>>
#
# Analytics dashboard for courses running on edx-platform.
# 
# Top-level module.

import logging
import os

import re
import json
import webapp2
import datetime

import gsdata
import bqutil
import auth
import local_config
import urllib

from unidecode import unidecode

from auth import auth_required
from stats import DataStats
from datatable import DataTableField
from datasource import DataSource
from dashboard import DashboardRoutes
from collections import defaultdict, OrderedDict

import jinja2

# from gviz_data_table import encode
# from gviz_data_table import Table

from google.appengine.api import memcache

mem = memcache.Client()

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
    extensions=['jinja2.ext.autoescape'],
    autoescape=True)

#-----------------------------------------------------------------------------

class MainPage(auth.AuthenticatedHandler, DataStats, DataSource):
    '''
    Main python class which displays views.
    '''

    @auth_required
    def get_main(self):
        '''
        Main page: show list of all courses
        '''
        courses = self.get_course_listings()

        # logging.info('course listings: %s' % courses['data'])

        html = self.list2table(map(DataTableField, 
                                   [{'field': 'launch', 'title':'Course launch', 'width': '12%'},
                                    {'field': 'course_number', 'title': 'Course #', 'width': '5%'}, 
                                    {'field': 'course_image', 'title': 'Course image'}, 
                                    {'field': 'title', 'title': 'Course Title', 'width': '40%'},
                                    {'field': 'course_id', 'title': 'course ID', 'width': '12%'},
                                   ]), 
                               courses['data'])

        data = self.common_data
        data.update({'data': {},
                     'is_staff': self.is_superuser(),
                     'table': html,
                 })
        template = JINJA_ENVIRONMENT.get_template('courses.html')
        self.response.out.write(template.render(data))

    @auth_required
    def get_admin(self):
        '''
        Admin page: show authorized users, clear cache
        '''
        if not self.user in self.AUTHORIZED_USERS:	# require superuser
            return self.no_auth_sorry()

        msg = ""
        action = self.request.POST.get('action', None)

        if action=='Flush cache':
            memcache.flush_all()
            msg = "Cache flushed"

        elif action=='Reload staff table':
            self.get_staff_table(reload=True)
            msg = "Staff table reloaded - please refresh this page"

        elif action=='Add staff':
            fields = ['username', 'role', 'course_id', 'notes']
            data = { x: (self.request.POST.get(x) or '') for x in fields }
            self.add_staff_table_entry(data)
            msg = "New staff %s added" % data

        todelete = self.request.POST.get('do-delete', None)
        if todelete is not None:
            self.disable_staff_table_entry(int(todelete))
            msg = "Deleted staff table row %s" % todelete

        stable = self.get_staff_table()

        stafftable = self.list2table([DataTableField({'icon':'delete', 'field': 'sid', 'title':' '}), 
                                      'username', 'role', 'course_id', 'notes'], stable)

        data = self.common_data
        data.update({'superusers': self.AUTHORIZED_USERS,
                     'table': stafftable,
                     'msg': msg,
                 })
        template = JINJA_ENVIRONMENT.get_template('admin.html')
        self.response.out.write(template.render(data))

    @auth_required
    def ajax_get_usage_stats(self, org=None, number=None, semester=None):

        course_id = '/'.join([org, number, semester])
        usdat = self.compute_usage_stats(course_id)['data'][0]
        # categories = ['registered', 'viewed', 'explored']
        data = {'data': [usdat] }

        # logging.info('ajax_get_usage_stats data=%s' % data)

        self.response.headers['Content-Type'] = 'application/json'   
        self.response.out.write(json.dumps(data))
        
    @auth_required
    def ajax_get_enrollment_stats(self, org=None, number=None, semester=None):
        '''
        Return enrollment stats in "series" format for HighCharts.
        Use course listings to determine start date
        '''
        course_id = '/'.join([org, number, semester])
        courses = self.get_course_listings()
        start = courses['data_by_key'][course_id]['launch']

        (y,m,d) = map(int, start.split('-'))
        start_dt = datetime.datetime(y, m, d)
        start_dt = start_dt - datetime.timedelta(days=32*9)	# start plot 9 months before launch
        start_str = start_dt.strftime('%Y-%m-%d')

        end_dt = start_dt + datetime.timedelta(days=32*1)	# default position for end selector
        end_str = end_dt.strftime('%Y-%m-%d')
        logging.info('initial start_dt=%s, end_dt=%s' % (start_dt, end_dt))

        bqdat = self.compute_enrollment_by_day(course_id, start=start_str)
        dtrange = ['2199-12-31', '1900-01-01']

        def getrow(x, field, scale):
            dtstr = x['date']
            if dtstr < dtrange[0]:	# min max dates from actual data
                dtrange[0] = dtstr
            if dtstr > dtrange[1]:
                dtrange[1] = dtstr
            ts = self.datetime2milliseconds(dtstr=dtstr)  # (dt - datetime.datetime(1970, 1, 1)).total_seconds() * 1000
            cnt = int(x[field])
            return [ts, cnt]

        def getseries(field):
            return {'name': field['name'], 'data': [ getrow(x, field['field'], field['scale']) for x in bqdat['data'] ]}

        def mkf(field, name, scale=1):
            return {'field': field, 'name': name, 'scale': scale}

        fields = [mkf('nenroll_cum', 'cumulative enrollment')]
        stats = [ getseries(sname) for sname in fields ]

        dtrange = [ datetime.datetime(*map(int, x.split('-'))) for x in dtrange ]
        start_dt = max(dtrange[0], start_dt)
        end_dt =  max(dtrange[0], end_dt)
        if end_dt == start_dt:
            end_dt = end_dt + datetime.timedelta(days=32*2)
        end_dt =  min(dtrange[1], end_dt)
        logging.info('dtrange=%s, actual start_dt=%s, end_dt=%s' % (dtrange, start_dt, end_dt))

        data = {'series': stats,
                'start_dt': self.datetime2milliseconds(start_dt),
                'end_dt': self.datetime2milliseconds(end_dt),
        }

        self.response.headers['Content-Type'] = 'application/json'   
        self.response.out.write(json.dumps(data))
        
    def setup_module_info(self, org=None, number=None, semester=None, url_name=None):
        '''
        setup common module info for problem, video, html
        '''
        course_id = '/'.join([org, number, semester])
        caxis = self.load_course_axis(course_id)

        # get problem info
        the_module = caxis[url_name]
        module_id = the_module['module_id']
        chapter_mid = the_module['chapter_mid']
        chapter_url_name = chapter_mid.rsplit('/',1)[-1]
        name = the_module['name']

        data = self.common_data
        data.update({'course_id': course_id,
                     'name': name,
                     'chapter_mid': chapter_mid,
                     'cun': chapter_url_name,
                     'base': self.get_base(course_id),
                     'url_name': url_name,
                     'module_id': module_id,
                     'module': the_module,
                     'caxis': caxis,
                 })
        return data

    @auth_required
    def get_problem(self, **kwargs):
        '''
        single problem analytics view

        - iframe
        '''

        data = self.setup_module_info(**kwargs)
        # template = os.path.join(os.path.dirname(__file__), 'problem.html')
        template = JINJA_ENVIRONMENT.get_template('problem.html')
        self.response.out.write(template.render(data))


    @auth_required
    def ajax_get_problem_stats(self, org=None, number=None, semester=None, problem_url_name=None):

        course_id = '/'.join([org, number, semester])

        ps = self.compute_problem_stats(course_id)
        # logging.info('problem url_name = %s' % problem_url_name)
        pstats = ps['data_by_key'].get(problem_url_name, [])
        # logging.info('pstats = %s' % pstats)

        data = {'data': [ pstats ] }

        self.response.headers['Content-Type'] = 'application/json'   
{'fraction_non_alphanumeric': 0.1368523390140684, 'fraction_numerical': 0.007777650691982157, 'mean_word_length': 2.1904761904761907, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4747179', 'n_tokens_mistral': 5434, 'n_tokens_neox': 5160, 'n_words': 1134}