<commit_before>from django.core.management.base import BaseCommand

from crawler.crawler import crawl_course, crawl_dept
try:
    from crawler.decaptcha import Entrance, DecaptchaFailure
except ImportError:
    Entrance = None
from data_center.models import Course, Department


def get_auth_pair(url):
    if Entrance is not None:
        try:
            return Entrance(url).get_ticket()
        except DecaptchaFailure:
            print 'Automated decaptcha failed.'
    else:
        print 'crawler.decaptcha not available (requires tesseract >= 3.03).'
    print 'Please provide valid ACIXSTORE and auth_num from'
    print url
    ACIXSTORE = raw_input('ACIXSTORE: ')
    auth_num = raw_input('auth_num: ')
    return ACIXSTORE, auth_num


class Command(BaseCommand):
    args = ''
    help = 'Help crawl the course data from NTHU.'

    def handle(self, *args, **kwargs):
        if len(args) == 0:
            # ACIXSTORE, auth_num = get_auth_pair(
                # 'https://www.ccxp.nthu.edu.tw/ccxp/INQUIRE'
                # '/JH/6/6.2/6.2.9/JH629001.php'
            # )
            # crawl_course()

            # ACIXSTORE, auth_num = get_auth_pair(
                # 'https://www.ccxp.nthu.edu.tw/ccxp/INQUIRE'
                # '/JH/6/6.2/6.2.3/JH623001.php'
            # )
            crawl_dept()
            # elapsed_time = time.time() - start_time
            # print 'Total %.3f second used.' % elapsed_time
        if len(args) == 1:
            if args[0] == 'clear':
                Course.objects.all().delete()
                Department.objects.all().delete()<commit_msg>Remove redudant code and enable crawling courses<commit_after>from django.core.management.base import BaseCommand

from crawler.crawler import crawl_course, crawl_dept
try:
    from crawler.decaptcha import Entrance, DecaptchaFailure
except ImportError:
    Entrance = None
from data_center.models import Course, Department


def get_auth_pair(url):
    if Entrance is not None:
        try:
            return Entrance(url).get_ticket()
        except DecaptchaFailure:
            print 'Automated decaptcha failed.'
    else:
        print 'crawler.decaptcha not available (requires tesseract >= 3.03).'
    print 'Please provide valid ACIXSTORE and auth_num from'
    print url
    ACIXSTORE = raw_input('ACIXSTORE: ')
    auth_num = raw_input('auth_num: ')
    return ACIXSTORE, auth_num


class Command(BaseCommand):
    args = ''
    help = 'Help crawl the course data from NTHU.'

    def handle(self, *args, **kwargs):
        if len(args) == 0:
            crawl_course()

            crawl_dept()
            # elapsed_time = time.time() - start_time
            # print 'Total %.3f second used.' % elapsed_time
        if len(args) == 1:
            if args[0] == 'clear':
                Course.objects.all().delete()
                Department.objects.all().delete()
{'fraction_non_alphanumeric': 0.10440513354144988, 'fraction_numerical': 0.013180714533472077, 'mean_word_length': 2.284738041002278, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4362096', 'n_tokens_mistral': 951, 'n_tokens_neox': 902, 'n_words': 219}