<commit_before>import logging, re
from pprint import pprint
from datetime import time
import pyodbc
from django.template.defaultfilters import slugify, 
from aspc.coursesearch.models import (Course, Meeting, Department, 
    RequirementArea, CAMPUSES, CAMPUSES_LOOKUP)

FEE_REGEX = re.compile(r'[Ff]ee:\s+\$([\d\.]+)')
ROOM_REGEX = re.compile(r'[A-Z]+\s([^(]+)\s+')

logger = logging.getLogger(__name__)

def _is_requirement_area(deptcode):
    # By convention, dept codes of the form "letter, digit, letter" are
    # actually requirement areas (like Pomona Gen Eds)
    if deptcode[1] in [str(a) for a in range(1, len(CAMPUSES) + 1)]:
        return True
    else:
        return False

def refresh_departments(cursor):
    <commit_msg>sorting out stashes more<commit_after>import logging, re
from pprint import pprint
from datetime import time
import pyodbc
from django.template.defaultfilters import slugify, 
from aspc.coursesearch.models import (Course, Meeting, Department, 
    RequirementArea, CAMPUSES, CAMPUSES_LOOKUP)

FEE_REGEX = re.compile(r'[Ff]ee:\s+\$([\d\.]+)')
ROOM_REGEX = re.compile(r'[A-Z]+\s([^(]+)\s+')

logger = logging.getLogger(__name__)

def _is_requirement_area(deptcode):
    # By convention, dept codes of the form "letter, digit, letter" are
    # actually requirement areas (like Pomona Gen Eds)
    if deptcode[1] in [str(a) for a in range(1, len(CAMPUSES) + 1)]:
        return True
    else:
        return False

        # get refreshes greater than last stored full
        # pull refreshes one at a time until we have a last full and a last reg
        # if last full == last stored full, ignore it
        # else store last full
        # if last reg == last stored reg, ignored it
        # else store last reg

def refresh_departments(cursor):
    
{'fraction_non_alphanumeric': 0.10726256983240223, 'fraction_numerical': 0.0033519553072625698, 'mean_word_length': 3.713157894736842, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7560833', 'n_tokens_mistral': 600, 'n_tokens_neox': 584, 'n_words': 188}