# Path: crawler/crawler.py
# def crawl_course(acixstore, auth_num, cou_codes, ys):
#     with FuturesSession(max_workers=MAX_WORKERS) as session:
#         curriculum_futures = [
#             cou_code_2_future(session, cou_code, acixstore, auth_num, ys)
#             for cou_code in cou_codes
#         ]
# 
#         progress = progressbar.ProgressBar(maxval=len(cou_codes))
#         with transaction.atomic():
#             for future, cou_code in progress(zip(curriculum_futures,
#                                                  cou_codes)):
#                 response = future.result()
#                 response.encoding = 'cp950'
#                 handle_curriculum_html(response.text, cou_code)
# 
#     print('Crawling syllabus...')
#     course_list = list(Course.objects.all())
# 
#     with FuturesSession(max_workers=MAX_WORKERS) as session:
#         course_futures = [
#             session.get(
#                 syllabus_url,
#                 params={
#                     'c_key': course.no,
#                     'ACIXSTORE': acixstore,
#                 }
#             )
#             for course in course_list
#         ]
# 
#         progress = progressbar.ProgressBar(maxval=len(course_list))
#         for future, course in progress(zip_longest(course_futures,
#                                                    course_list)):
#             response = future.result()
#             response.encoding = 'cp950'
#             save_syllabus(response.text, course, ys)
# 
#         print('Total course information: %d' % Course.objects.filter(ys=ys).count())  # noqa
# 
# def crawl_dept(acixstore, auth_num, dept_codes, ys):
#     with FuturesSession(max_workers=MAX_WORKERS) as session:
#         future_depts = [
#             dept_2_future(session, dept_code, acixstore, auth_num, ys)
#             for dept_code in dept_codes
#         ]
# 
#         progress = progressbar.ProgressBar()
#         with transaction.atomic():
#             for future in progress(future_depts):
#                 response = future.result()
#                 response.encoding = encoding
#                 handle_dept_html(response.text, ys)
# 
#     print('Total department information: %d' % Department.objects.filter(ys=ys).count())  # noqa
# 
# Path: crawler/course.py
# def get_cou_codes():
#     html = get(form_url).text
#     document = lxml.html.fromstring(html)
#     return document.xpath('//select[@name="cou_code"]/option/@value')
# 
# Path: data_center/models.py
# class Course(models.Model):
#     """Course database schema"""
#     no = models.CharField(max_length=20, unique=True, db_index=True)
#     code = models.CharField(max_length=20, blank=True)
#     eng_title = models.CharField(max_length=200, blank=True)
#     chi_title = models.CharField(max_length=200, blank=True)
#     note = models.TextField(blank=True)
#     objective = models.CharField(max_length=80, blank=True)
#     time = models.CharField(max_length=20, blank=True)
#     time_token = models.CharField(max_length=20, blank=True)
#     teacher = models.CharField(max_length=40, blank=True)  # Only save Chinese
#     room = models.CharField(max_length=80, blank=True)
#     credit = models.IntegerField(default=0)
#     limit = models.IntegerField(default=0)
#     prerequisite = models.BooleanField(default=False, blank=True)
#     ys = models.CharField(max_length=10, blank=True)
# 
#     ge = models.CharField(max_length=80, blank=True)
# 
#     hit = models.IntegerField(default=0)
# 
#     syllabus = models.TextField(blank=True)  # pure text
#     has_attachment = models.BooleanField(default=False)  # has pdf
# 
#     def __str__(self):
#         return self.no
# 
#     @property
#     def attachment_url(self):
#         return attachment_url_format % urlquote(self.no)
# 
# class Department(models.Model):
#     dept_name = models.CharField(max_length=20, blank=True)
#     required_course = models.ManyToManyField(Course, blank=True)
#     ys = models.CharField(max_length=10, blank=True)
# 
#     def __str__(self):
#         return self.dept_name
# 
# Path: utils/config.py
# def get_config(section, option, filename='nthu_course.cfg'):
#     '''Return a config in that section'''
#     try:
#         config = configparser.ConfigParser()
#         config.optionxform = str
#         config.read(settings.BASE_DIR + '/NTHU_Course/config/' + filename)
#         return config.get(section, option)
#     except Exception as ex:
#         # no config found
#         return None


from django.core.management.base import BaseCommand
from crawler.crawler import crawl_course, crawl_dept
from crawler.course import get_cou_codes
    from crawler.decaptcha import Entrance, DecaptchaFailure
from data_center.models import Course, Department
from utils.config import get_config
            import time

try:
except ImportError:
    Entrance = None


def get_auth_pair(url):
    if Entrance is not None:
        try:
            return Entrance(url).get_ticket()
        except DecaptchaFailure:
            print('Automated decaptcha failed.')
    else:
        print('crawler.decaptcha not available (requires tesseract >= 3.03).')

# Path: crawler/crawler.py
# def crawl_course(acixstore, auth_num, cou_codes, ys):
#     with FuturesSession(max_workers=MAX_WORKERS) as session:
#         curriculum_futures = [
#             cou_code_2_future(session, cou_code, acixstore, auth_num, ys)
#             for cou_code in cou_codes
#         ]
# 
#         progress = progressbar.ProgressBar(maxval=len(cou_codes))
#         with transaction.atomic():
#             for future, cou_code in progress(zip(curriculum_futures,
#                                                  cou_codes)):
#                 response = future.result()
#                 response.encoding = 'cp950'
#                 handle_curriculum_html(response.text, cou_code)
# 
#     print('Crawling syllabus...')
#     course_list = list(Course.objects.all())
# 
#     with FuturesSession(max_workers=MAX_WORKERS) as session:
#         course_futures = [
#             session.get(
#                 syllabus_url,
#                 params={
#                     'c_key': course.no,
#                     'ACIXSTORE': acixstore,
#                 }
#             )
#             for course in course_list
#         ]
# 
#         progress = progressbar.ProgressBar(maxval=len(course_list))
#         for future, course in progress(zip_longest(course_futures,
#                                                    course_list)):
#             response = future.result()
#             response.encoding = 'cp950'
#             save_syllabus(response.text, course, ys)
# 
#         print('Total course information: %d' % Course.objects.filter(ys=ys).count())  # noqa
# 
# def crawl_dept(acixstore, auth_num, dept_codes, ys):
#     with FuturesSession(max_workers=MAX_WORKERS) as session:
#         future_depts = [
#             dept_2_future(session, dept_code, acixstore, auth_num, ys)
#             for dept_code in dept_codes
#         ]
# 
#         progress = progressbar.ProgressBar()
#         with transaction.atomic():
#             for future in progress(future_depts):
#                 response = future.result()
#                 response.encoding = encoding
#                 handle_dept_html(response.text, ys)
# 
#     print('Total department information: %d' % Department.objects.filter(ys=ys).count())  # noqa
# 
# Path: crawler/course.py
# def get_cou_codes():
#     html = get(form_url).text
#     document = lxml.html.fromstring(html)
#     return document.xpath('//select[@name="cou_code"]/option/@value')
# 
# Path: data_center/models.py
# class Course(models.Model):
#     """Course database schema"""
#     no = models.CharField(max_length=20, unique=True, db_index=True)
#     code = models.CharField(max_length=20, blank=True)
#     eng_title = models.CharField(max_length=200, blank=True)
#     chi_title = models.CharField(max_length=200, blank=True)
#     note = models.TextField(blank=True)
#     objective = models.CharField(max_length=80, blank=True)
#     time = models.CharField(max_length=20, blank=True)
#     time_token = models.CharField(max_length=20, blank=True)
#     teacher = models.CharField(max_length=40, blank=True)  # Only save Chinese
#     room = models.CharField(max_length=80, blank=True)
#     credit = models.IntegerField(default=0)
#     limit = models.IntegerField(default=0)
#     prerequisite = models.BooleanField(default=False, blank=True)
#     ys = models.CharField(max_length=10, blank=True)
# 
#     ge = models.CharField(max_length=80, blank=True)
# 
#     hit = models.IntegerField(default=0)
# 
#     syllabus = models.TextField(blank=True)  # pure text
#     has_attachment = models.BooleanField(default=False)  # has pdf
# 
#     def __str__(self):
#         return self.no
# 
#     @property
#     def attachment_url(self):
#         return attachment_url_format % urlquote(self.no)
# 
# class Department(models.Model):
#     dept_name = models.CharField(max_length=20, blank=True)
#     required_course = models.ManyToManyField(Course, blank=True)
#     ys = models.CharField(max_length=10, blank=True)
# 
#     def __str__(self):
#         return self.dept_name
# 
# Path: utils/config.py
# def get_config(section, option, filename='nthu_course.cfg'):
#     '''Return a config in that section'''
#     try:
#         config = configparser.ConfigParser()
#         config.optionxform = str
#         config.read(settings.BASE_DIR + '/NTHU_Course/config/' + filename)
#         return config.get(section, option)
#     except Exception as ex:
#         # no config found
#         return None


# Path: crawler/management/commands/crawl_course.py
from django.core.management.base import BaseCommand
from crawler.crawler import crawl_course, crawl_dept
from crawler.course import get_cou_codes
    from crawler.decaptcha import Entrance, DecaptchaFailure
from data_center.models import Course, Department
from utils.config import get_config
            import time


try:
except ImportError:
    Entrance = None


def get_auth_pair(url):
    if Entrance is not None:
        try:
            return Entrance(url).get_ticket()
        except DecaptchaFailure:
            print('Automated decaptcha failed.')
    else:
        print('crawler.decaptcha not available (requires tesseract >= 3.03).')

    print('Please provide valid ACIXSTORE and auth_num from')