# Path: apps/spider/crawlers/medians.py
# MEDIAN_PAGE_INDEX_URL = "http://www.dartmouth.edu/reg/transcript/medians/"
# MEDIANS_URL_FMT = (
#     "http://www.dartmouth.edu/reg/transcript/medians/{term}.html")
# def get_term_from_median_page_url(url):
# def crawl_median_page_urls():
# def _retrieve_term_medians_urls_from_soup(soup):
# def _is_term_page_url(url):
# def crawl_term_medians_for_url(url):
# def _median_dict_sorter(a, b):
# def _convert_table_row_to_dict(table_row):
# def import_medians(data):
# def import_median(median_data):
# 
# Path: apps/spider/utils.py
# DEPARTMENT_CORRECTIONS = {
#     "M&SS": "QSS",
#     "WGST": "WGSS"
# }
# def clean_department_code(department):
# def int_or_none(string):
# def pretty_json(data):
# def parse_number_and_subnumber(numbers_text):
# def retrieve_soup(url, data=None, preprocess=lambda x: x):
# 
# Path: lib/constants.py
# CURRENT_TERM = os.environ["CURRENT_TERM"]  # e.g. 16S
# SUPPORT_EMAIL = "support@layuplist.com"
# REC_UPVOTE_REQ = 2
# OFFERINGS_THRESHOLD_FOR_TERM_UPDATE = int(
#     os.environ["OFFERINGS_THRESHOLD_FOR_TERM_UPDATE"])


import difflib
import json
from django.conf import settings
from django.contrib.postgres.fields import JSONField
from django.core.mail import send_mail
from django.db import models, transaction
from apps.spider.crawlers import medians
from apps.spider import utils
from lib import constants
        from apps.spider.tasks import import_pending_crawled_data

    resource = models.CharField(max_length=128, db_index=True, unique=True)
    data_type = models.CharField(max_length=32, choices=DATA_TYPE_CHOICES)
    pending_data = JSONField()
    current_data = JSONField(null=True)

    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def __unicode__(self):
        return "[{data_type}] {resource}".format(
            data_type=self.data_type,
            resource=self.resource,
        )

    def has_change(self):
        return self.pending_data != self.current_data

    @property
    def diff(self):
        if not self.current_data:
            return utils.pretty_json(self.pending_data)
        if self.has_change:
            return "\n".join(difflib.unified_diff(
                utils.pretty_json(self.current_data).splitlines(),
                utils.pretty_json(self.pending_data).splitlines(),
            ))

    @property
    def pretty_current_data(self):

# Path: apps/spider/crawlers/medians.py
# MEDIAN_PAGE_INDEX_URL = "http://www.dartmouth.edu/reg/transcript/medians/"
# MEDIANS_URL_FMT = (
#     "http://www.dartmouth.edu/reg/transcript/medians/{term}.html")
# def get_term_from_median_page_url(url):
# def crawl_median_page_urls():
# def _retrieve_term_medians_urls_from_soup(soup):
# def _is_term_page_url(url):
# def crawl_term_medians_for_url(url):
# def _median_dict_sorter(a, b):
# def _convert_table_row_to_dict(table_row):
# def import_medians(data):
# def import_median(median_data):
# 
# Path: apps/spider/utils.py
# DEPARTMENT_CORRECTIONS = {
#     "M&SS": "QSS",
#     "WGST": "WGSS"
# }
# def clean_department_code(department):
# def int_or_none(string):
# def pretty_json(data):
# def parse_number_and_subnumber(numbers_text):
# def retrieve_soup(url, data=None, preprocess=lambda x: x):
# 
# Path: lib/constants.py
# CURRENT_TERM = os.environ["CURRENT_TERM"]  # e.g. 16S
# SUPPORT_EMAIL = "support@layuplist.com"
# REC_UPVOTE_REQ = 2
# OFFERINGS_THRESHOLD_FOR_TERM_UPDATE = int(
#     os.environ["OFFERINGS_THRESHOLD_FOR_TERM_UPDATE"])


# Path: apps/spider/models.py
import difflib
import json
from django.conf import settings
from django.contrib.postgres.fields import JSONField
from django.core.mail import send_mail
from django.db import models, transaction
from apps.spider.crawlers import medians
from apps.spider import utils
from lib import constants
        from apps.spider.tasks import import_pending_crawled_data


    resource = models.CharField(max_length=128, db_index=True, unique=True)
    data_type = models.CharField(max_length=32, choices=DATA_TYPE_CHOICES)
    pending_data = JSONField()
    current_data = JSONField(null=True)

    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def __unicode__(self):
        return "[{data_type}] {resource}".format(
            data_type=self.data_type,
            resource=self.resource,
        )

    def has_change(self):
        return self.pending_data != self.current_data

    @property
    def diff(self):
        if not self.current_data:
            return utils.pretty_json(self.pending_data)
        if self.has_change:
            return "\n".join(difflib.unified_diff(
                utils.pretty_json(self.current_data).splitlines(),
                utils.pretty_json(self.pending_data).splitlines(),
            ))

    @property
    def pretty_current_data(self):

        return utils.pretty_json(self.current_data)