<commit_before>#
# migration tools for content team to go from stable-edx4edx to LMS+CMS
#

import json
import logging
import os
from pprint import pprint
import xmodule.modulestore.django as xmodule_django
from xmodule.modulestore.django import modulestore

from django.http import HttpResponse
from django.conf import settings
import track.views

try:
    from django.views.decorators.csrf import csrf_exempt
except ImportError:
    from django.contrib.csrf.middleware import csrf_exempt

log = logging.getLogger("mitx.lms_migrate")
LOCAL_DEBUG = True
ALLOWED_IPS = settings.LMS_MIGRATION_ALLOWED_IPS

def escape(s):
    """escape HTML special characters in string"""
    return str(s).replace('<','&lt;').replace('>','&gt;')

def getip(request):
    '''
    Extract IP address of requester from header, even if behind proxy
    '''
    ip = request.META.get('HTTP_X_REAL_IP','')	# nginx reverse proxy
    if not ip:
        ip = request.META.get('REMOTE_ADDR','None')
    return ip


def get_commit_id(course):
    return course.metadata.get('GIT_COMMIT_ID','No commit id')
    # getattr(def_ms.courses[reload_dir], 'GIT_COMMIT_ID','No commit id')


def set_commit_id(course,commit_id):
    course.metadata['GIT_COMMIT_ID'] = commit_id
    # setattr(def_ms.courses[reload_dir], 'GIT_COMMIT_ID', new_commit_id)

def manage_modulestores(request, reload_dir=None, commit_id=None):
    '''
    Manage the static in-memory modulestores.

    If reload_dir is not None, then instruct the xml loader to reload that course directory.
    '''
    html = "<html><body>"

    def_ms = modulestore()
    courses = def_ms.get_courses()

    #----------------------------------------
    # check on IP address of requester

    ip = getip(request)

    if LOCAL_DEBUG:
        html += '<h3>IP address: %s <h3>' % ip
        html += '<h3>User: %s </h3>' % request.user
        html += '<h3>My pid: %s</h3>' % os.getpid()
        log.debug('request from ip=%s, user=%s' % (ip,request.user))

    if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
        if request.user and request.user.is_staff:
            log.debug('request allowed because user=%s is staff' % request.user)
        else:
            html += 'Permission denied'
            html += "</body></html>"
            log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS)
            return HttpResponse(html, status=403)

    #----------------------------------------
    # reload course if specified; handle optional commit_id

    if reload_dir is not None:
        if reload_dir not in def_ms.courses:
            html += '<h2 class="inline-error">Error: "%s" is not a valid course directory</h2>' % reload_dir
        else:
            # reloading based on commit_id is needed when running mutiple worker threads,
            # so that a given thread doesn't reload the same commit multiple times
            current_commit_id = get_commit_id(def_ms.courses[reload_dir])
            log.debug('commit_id="%s"' % commit_id)
            log.debug('current_commit_id="%s"' % current_commit_id)

            if (commit_id is not None) and (commit_id==current_commit_id):
                html += "<h2>Already at commit id %s for %s</h2>" % (commit_id, reload_dir)
            else:
                html += '<h2>Reloaded course directory "%s"</h2>' % reload_dir
                def_ms.try_load_course(reload_dir)
                gdir = settings.DATA_DIR / reload_dir 
                new_commit_id = os.popen('cd %s; git log -n 1 | head -1' % gdir).read().strip().split(' ')[1]
                set_commit_id(def_ms.courses[reload_dir], new_commit_id)
                html += '<p>commit_id=%s</p>' % new_commit_id

    #----------------------------------------

    html += '<h2>Courses loaded in the modulestore</h2>'
    html += '<ol>'
    for cdir, course in def_ms.courses.items():
        html += '<li><a href="%s/migrate/reload/%s">%s</a> (%s)</li>' % (settings.MITX_ROOT_URL,
                                                            escape(cdir),
                                                            escape(cdir),
                                                            course.location.url())
    html += '</ol>'

    #----------------------------------------

    dumpfields = ['definition','location','metadata']

    for cdir, course in def_ms.courses.items():
        html += '<hr width="100%"/>'
        html += '<h2>Course: %s (%s)</h2>' % (course.display_name,cdir)

        html += '<p>commit_id=%s</p>' % get_commit_id(course)

        for field in dumpfields:
            data = getattr(course,field)
            html += '<h3>%s</h3>' % field
            if type(data)==dict:
                html += '<ul>'<commit_msg>add tracking log entries for modulestore reload<commit_after>#
# migration tools for content team to go from stable-edx4edx to LMS+CMS
#

import json
import logging
import os
from pprint import pprint
import xmodule.modulestore.django as xmodule_django
from xmodule.modulestore.django import modulestore

from django.http import HttpResponse
from django.conf import settings
import track.views

try:
    from django.views.decorators.csrf import csrf_exempt
except ImportError:
    from django.contrib.csrf.middleware import csrf_exempt

log = logging.getLogger("mitx.lms_migrate")
LOCAL_DEBUG = True
ALLOWED_IPS = settings.LMS_MIGRATION_ALLOWED_IPS

def escape(s):
    """escape HTML special characters in string"""
    return str(s).replace('<','&lt;').replace('>','&gt;')

def getip(request):
    '''
    Extract IP address of requester from header, even if behind proxy
    '''
    ip = request.META.get('HTTP_X_REAL_IP','')	# nginx reverse proxy
    if not ip:
        ip = request.META.get('REMOTE_ADDR','None')
    return ip


def get_commit_id(course):
    return course.metadata.get('GIT_COMMIT_ID','No commit id')
    # getattr(def_ms.courses[reload_dir], 'GIT_COMMIT_ID','No commit id')


def set_commit_id(course,commit_id):
    course.metadata['GIT_COMMIT_ID'] = commit_id
    # setattr(def_ms.courses[reload_dir], 'GIT_COMMIT_ID', new_commit_id)

def manage_modulestores(request, reload_dir=None, commit_id=None):
    '''
    Manage the static in-memory modulestores.

    If reload_dir is not None, then instruct the xml loader to reload that course directory.
    '''
    html = "<html><body>"

    def_ms = modulestore()
    courses = def_ms.get_courses()

    #----------------------------------------
    # check on IP address of requester

    ip = getip(request)

    if LOCAL_DEBUG:
        html += '<h3>IP address: %s <h3>' % ip
        html += '<h3>User: %s </h3>' % request.user
        html += '<h3>My pid: %s</h3>' % os.getpid()
        log.debug('request from ip=%s, user=%s' % (ip,request.user))

    if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
        if request.user and request.user.is_staff:
            log.debug('request allowed because user=%s is staff' % request.user)
        else:
            html += 'Permission denied'
            html += "</body></html>"
            log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS)
            return HttpResponse(html, status=403)

    #----------------------------------------
    # reload course if specified; handle optional commit_id

    if reload_dir is not None:
        if reload_dir not in def_ms.courses:
            html += '<h2 class="inline-error">Error: "%s" is not a valid course directory</h2>' % reload_dir
        else:
            # reloading based on commit_id is needed when running mutiple worker threads,
            # so that a given thread doesn't reload the same commit multiple times
            current_commit_id = get_commit_id(def_ms.courses[reload_dir])
            log.debug('commit_id="%s"' % commit_id)
            log.debug('current_commit_id="%s"' % current_commit_id)

            if (commit_id is not None) and (commit_id==current_commit_id):
                html += "<h2>Already at commit id %s for %s</h2>" % (commit_id, reload_dir)
                track.views.server_track(request,
                                         'reload %s skipped already at %s (pid=%s)' % (reload_dir,
                                                                                       commit_id,
                                                                                       os.getpid(),
                                                                                       ),
                                         {}, page='migrate')
            else:
                html += '<h2>Reloaded course directory "%s"</h2>' % reload_dir
                def_ms.try_load_course(reload_dir)
                gdir = settings.DATA_DIR / reload_dir 
                new_commit_id = os.popen('cd %s; git log -n 1 | head -1' % gdir).read().strip().split(' ')[1]
                set_commit_id(def_ms.courses[reload_dir], new_commit_id)
                html += '<p>commit_id=%s</p>' % new_commit_id
                track.views.server_track(request, 'reloaded %s now at %s (pid=%s)' % (reload_dir,
                                                                                      new_commit_id,
                                                                                      os.getpid()), {}, page='migrate')

    #----------------------------------------

    html += '<h2>Courses loaded in the modulestore</h2>'
    html += '<ol>'
    for cdir, course in def_ms.courses.items():
        html += '<li><a href="%s/migrate/reload/%s">%s</a> (%s)</li>' % (settings.MITX_ROOT_URL,
                                                            escape(cdir),
                                                            escape(cdir),
                                                            course.location.url())
    html += '</ol>'

    #----------------------------------------

    dumpfields = ['definition','location','metadata']

    for cdir, course in def_ms.courses.items():
        html += '<hr width="100%"/>'
        html += '<h2>Course: %s (%s)</h2>' % (course.display_name,cdir)

        html += '<p>commit_id=%s</p>' % get_commit_id(course)

        for field in dumpfields:
            data = getattr(course,field)
            html += '<h3>%s</h3>' % field
            if type(data)==dict:
                html += '<ul>'
{'fraction_non_alphanumeric': 0.1494589061128985, 'fraction_numerical': 0.005459686068051087, 'mean_word_length': 1.9258414147176268, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 35, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7044896', 'n_tokens_mistral': 3056, 'n_tokens_neox': 2950, 'n_words': 746}