<commit_before>#!/usr/bin/python
#
# File:   git2edx.py
# Date:   12-Mar-14
# Author: <NAME> <<EMAIL>>
#
# web server which provides git2edx service
# uses wsgi
#
# configuration parameters in config.py

import sys, string, re, time, os
import cgi
import json
import urllib
import requests
import socket

from cgi import parse_qs, escape
from lxml import etree
from edxStudio import edxStudio

#-----------------------------------------------------------------------------
# config

config = {'username': "",
          'password': "",
          'ANYREPO': True,
          'FORCE_BRANCH': '',
          'REPODIR': '',
          'LOGFILE': "",
          'REPO2COURSE_MAP': {},	# if empty, then course id will be determined from repo XML contents
          'PORT': 8121,
          }

CFN = 'config.json'
if os.path.exists(CFN):
    import json
    new_config = json.loads(open(CFN).read())
    config.update(new_config)

#-----------------------------------------------------------------------------

PIDFILE = "git2edx.pid"

open(PIDFILE,'w').write(str(os.getpid()))

def LOG(x):
    fp = open(config['LOGFILE'],'a')
    if type(x)==dict:
        for k in x:
            if not k:
                continue
            s = '  %s : %s' % (k,x[k])
            fp.write(s)
            print s
    #if type(x)==type('str'):
    else:
        fp.write(x)
        fp.write('\n')
        print x

    sys.stdout.flush()
    fp.flush()
    fp.close()

#-----------------------------------------------------------------------------

def upload_to_edx(rdir, repo):
    '''
    tar up repo, and upload to edX

    rdir = directory with repo contents
    repo = repo name
    '''
    
    # get course_id
    if not config['REPO2COURSE_MAP']:
        cxml = etree.parse(open('%s/course.xml' % rdir)).getroot()
        org = cxml.get('org')
        course = cxml.get('course')
        sem = cxml.get('url_name')
        course_id = '/'.join([org, course, sem])
    else:
        course_id = config['REPO2COURSE_MAP'].get(repo, '')
        
    if not course_id:
        LOG("Error: cannot determine course_id for repo=%s" % repo)
        return
    
    # create tar.gz file
    tfn = '%s.tar.gz' % rdir
    os.chdir(rdir)
    os.chdir('..')
    cmd = "tar czf %s --exclude=.git --exclude=src %s" % (tfn, repo)
    LOG(cmd)
    LOG(os.popen(cmd).read())
    
    # upload to studio
    LOG('-'*30 + "Uploading %s to edX studio course_id=%s" % (tfn, course_id))
    es = edxStudio(username=config['username'], password=config['password'])
    es.do_upload(course_id, tfn, nwait=3)

#-----------------------------------------------------------------------------

def do_git2edx(environ, start_response):<commit_msg>allow site to be specified for repo<commit_after>#!/usr/bin/python
#
# File:   git2edx.py
# Date:   12-Mar-14
# Author: <NAME> <<EMAIL>>
#
# web server which provides git2edx service
# uses wsgi
#
# configuration parameters in config.py

import sys, string, re, time, os
import cgi
import json
import urllib
import requests
import socket

from cgi import parse_qs, escape
from lxml import etree
from edxStudio import edxStudio

#-----------------------------------------------------------------------------
# config

config = {'username': "",
          'password': "",
          'ANYREPO': True,
          'FORCE_BRANCH': '',
          'REPODIR': '',
          'LOGFILE': "",
          'REPO2COURSE_MAP': {},	# if empty, then course id will be determined from repo XML contents
          'PORT': 8121,
          }

CFN = 'config.json'
if os.path.exists(CFN):
    import json
    new_config = json.loads(open(CFN).read())
    config.update(new_config)

#-----------------------------------------------------------------------------

PIDFILE = "git2edx.pid"

open(PIDFILE,'w').write(str(os.getpid()))

def LOG(x):
    fp = open(config['LOGFILE'],'a')
    if type(x)==dict:
        for k in x:
            if not k:
                continue
            s = '  %s : %s' % (k,x[k])
            fp.write(s)
            print s
    #if type(x)==type('str'):
    else:
        fp.write(x)
        fp.write('\n')
        print x

    sys.stdout.flush()
    fp.flush()
    fp.close()

#-----------------------------------------------------------------------------

def upload_to_edx(rdir, repo):
    '''
    tar up repo, and upload to edX

    rdir = directory with repo contents
    repo = repo name
    '''
    
    site_url = "https://studio.edx.org"

    # get course_id (and optional site_url)
    if not config['REPO2COURSE_MAP']:
        cxml = etree.parse(open('%s/course.xml' % rdir)).getroot()
        org = cxml.get('org')
        course = cxml.get('course')
        sem = cxml.get('url_name')
        course_id = '/'.join([org, course, sem])
    else:
        r2c = config['REPO2COURSE_MAP'].get(repo, '')
        if isinstance(r2c, dict):
            course_id = r2c['cid']
            site_url = r2c['site']
        else:
            course_id = r2c
        
    if not course_id:
        LOG("Error: cannot determine course_id for repo=%s" % repo)
        return
    
    # create tar.gz file
    tfn = '%s.tar.gz' % rdir
    os.chdir(rdir)
    os.chdir('..')
    cmd = "tar czf %s --exclude=.git --exclude=src %s" % (tfn, repo)
    LOG(cmd)
    LOG(os.popen(cmd).read())
    
    # upload to studio
    LOG('-'*30 + "Uploading %s to edX studio course_id=%s" % (tfn, course_id))
    es = edxStudio(username=config['username'], password=config['password'], base=site_url)
    es.do_upload(course_id, tfn, nwait=3)

#-----------------------------------------------------------------------------

def do_git2edx(environ, start_response):
{'fraction_non_alphanumeric': 0.2427253371185238, 'fraction_numerical': 0.007274662881476224, 'mean_word_length': 1.6464788732394366, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4364525', 'n_tokens_mistral': 1834, 'n_tokens_neox': 1799, 'n_words': 435}