<commit_before>import json
import re
import html
import logging
import os
from textwrap import dedent

from .break_apart_departments import break_apart_departments
from .check_for_course_revisions import check_for_revisions
from .data import course_types
from .parse_links_for_text import parse_links_for_text
from .parse_paragraph_as_list import parse_paragraph_as_list
from .paths import make_course_path
from .save_data import save_data
from .split_and_flip_instructors import split_and_flip_instructors


def json_date_handler(obj):
    if hasattr(obj, 'isoformat'):
        return obj.isoformat()
    else:
        raise TypeError(f'Object of type {type(obj)} with value of {repr(obj)} is not JSON serializable')


def save_course(course):
    course_path = make_course_path(course['clbid'])
    json_course_data = json.dumps(course,
                                  indent='\t',
                                  default=json_date_handler,
                                  ensure_ascii=False,
                                  sort_keys=True) + '\n'
    save_data(json_course_data, course_path)


def check_for_course_file_existence(clbid):
    return os.path.exists(make_course_path(clbid))


def extract_notes(course):
    if course['notes'] and 'Will also meet' in course['notes']:
        info = dedent(f'''
            [{course['year']}{course['semester']}] {course['type'][0]} ({'/'.join(course['departments'])} {course['number']} | {course['clbid']} {course['crsid']}):
            \t{course['notes']}
            \t{course['times']} {course['locations']}
        ''')

        # get the timestring and location string out of the notes field
        notes_into_time_and_location_regex = r'.*meet ([MTWF][/-]?.*) in (.*)\.'
        results = re.search(notes_into_time_and_location_regex,
                            course['notes'])
        extra_times, extra_locations = results.groups()
        # print(info + '\n\t' + 'regex matches:', [extra_times, extra_locations])
        print(extra_times)

        # split_time_regex =

        split_location_regex = r'(\w+ ?\d+)(?: or ?(\w+ ?\d+))?'

        # expandedDays = {
        #   'M':  'Mo',
        #   'T':  'Tu',
        #   'W':  'We',
        #   'Th': 'Th',
        #   'F':  'Fr'
        # }

        # listOfDays = []

        # if '-' in daystring:
        #   # M-F, M-Th, T-F
        #   sequence = ['M', 'T', 'W', 'Th', 'F']
        #   startDay = daystring.split('-')[0]
        #   endDay = daystring.split('-')[1]
        #   listOfDays = sequence.slice(
        #       sequence.indexOf(startDay),
        #       sequence.indexOf(endDay) + 1
        #   )
        # else:
        #   # MTThFW
        #   spacedOutDays = daystring.replace(/([a-z]*)([A-Z])/g, '$1 $2')
        #   # The regex sticks an extra space at the front. trim() it.
        #   spacedOutDays = spacedOutDays.trim()
        #   listOfDays = spacedOutDays.split(' ')

        # # 'M' => 'Mo'
        # return list(map(lambda day: expandedDays[day], listOfDays))


def parse_prerequisites(course):
    search_str = 'Prereq'

    prereqs_list = [para[para.index(search_str):]
                      for para in course.get('description', [])
                      if search_str in para]

    if not prereqs_list:
        prereqs_list = [note[note.index(search_str):]
                          for note in course.get('notes', [])
                          if search_str in note]

    if prereqs_list:
        return "\n".join(prereqs_list)\
            .replace('Prerequisite: ', '')

    return False


def clean_course(course):
    # Unescape &amp; in course names
    course['name'] = html.unescape(course['coursename'])
    del course['coursename']

    course['section'] = course['coursesection']
    del course['coursesection']

    course['status'] = course['coursestatus']
    del course['coursestatus']

    # Remove <br> tags from the 'notes' field.
    if course['notes']:
        course['notes'] = course['notes'].split('<br>')
        course['notes'] = [' '.join(note.split()) for note in course['notes']]
        course['notes'] = [html.unescape(note) for note in course['notes']]
        course['notes'] = [note for note in course['notes'] if note]

    # Remove the unused varcredits property
    del course['varcredits']

    # Flesh out coursesubtype
    if course['coursesubtype'] and course['coursesubtype'] in course_types:
        course['type'] = course_types[course['coursesubtype']]
    else:
        course['type'] = course['coursesubtype']
        print(f"'{course['type']}' doesn't appear in the types list.")
    del course['coursesubtype']

    # Break apart dept names into lists
    course['departments'] = break_apart_departments(course['deptname'])
    del course['deptname']

    # Turn numbers into numbers
    course['clbid'] = int(course['clbid'])

    if re.search(r'\d{3}$', course['coursenumber']):
        course['number'] = int(course['coursenumber'])
    elif re.match(r'\dXX', course['coursenumber']):
        course['number'] = course['coursenumber']
    elif re.match(r'\d{3}I', course['coursenumber']):
        course['number'] = int(course['coursenumber'][:-1])
    else:
        course['number'] = course['coursenumber']
    del course['coursenumber']

    course['credits'] = float(course['credits'])
    course['crsid'] = int(course['crsid'])
    if course['groupid']:
        course['groupid'] = int(course['groupid'])

    # Turn booleans into booleans
    course['pn'] = True if course['pn'] is 'Y' else False
<commit_msg>convert enroll and max to integers<commit_after>import json
import re
import html
import logging
import os
from textwrap import dedent

from .break_apart_departments import break_apart_departments
from .check_for_course_revisions import check_for_revisions
from .data import course_types
from .parse_links_for_text import parse_links_for_text
from .parse_paragraph_as_list import parse_paragraph_as_list
from .paths import make_course_path
from .save_data import save_data
from .split_and_flip_instructors import split_and_flip_instructors


def json_date_handler(obj):
    if hasattr(obj, 'isoformat'):
        return obj.isoformat()
    else:
        raise TypeError(f'Object of type {type(obj)} with value of {repr(obj)} is not JSON serializable')


def save_course(course):
    course_path = make_course_path(course['clbid'])
    json_course_data = json.dumps(course,
                                  indent='\t',
                                  default=json_date_handler,
                                  ensure_ascii=False,
                                  sort_keys=True) + '\n'
    save_data(json_course_data, course_path)


def check_for_course_file_existence(clbid):
    return os.path.exists(make_course_path(clbid))


def extract_notes(course):
    if course['notes'] and 'Will also meet' in course['notes']:
        info = dedent(f'''
            [{course['year']}{course['semester']}] {course['type'][0]} ({'/'.join(course['departments'])} {course['number']} | {course['clbid']} {course['crsid']}):
            \t{course['notes']}
            \t{course['times']} {course['locations']}
        ''')

        # get the timestring and location string out of the notes field
        notes_into_time_and_location_regex = r'.*meet ([MTWF][/-]?.*) in (.*)\.'
        results = re.search(notes_into_time_and_location_regex,
                            course['notes'])
        extra_times, extra_locations = results.groups()
        # print(info + '\n\t' + 'regex matches:', [extra_times, extra_locations])
        print(extra_times)

        # split_time_regex =

        split_location_regex = r'(\w+ ?\d+)(?: or ?(\w+ ?\d+))?'

        # expandedDays = {
        #   'M':  'Mo',
        #   'T':  'Tu',
        #   'W':  'We',
        #   'Th': 'Th',
        #   'F':  'Fr'
        # }

        # listOfDays = []

        # if '-' in daystring:
        #   # M-F, M-Th, T-F
        #   sequence = ['M', 'T', 'W', 'Th', 'F']
        #   startDay = daystring.split('-')[0]
        #   endDay = daystring.split('-')[1]
        #   listOfDays = sequence.slice(
        #       sequence.indexOf(startDay),
        #       sequence.indexOf(endDay) + 1
        #   )
        # else:
        #   # MTThFW
        #   spacedOutDays = daystring.replace(/([a-z]*)([A-Z])/g, '$1 $2')
        #   # The regex sticks an extra space at the front. trim() it.
        #   spacedOutDays = spacedOutDays.trim()
        #   listOfDays = spacedOutDays.split(' ')

        # # 'M' => 'Mo'
        # return list(map(lambda day: expandedDays[day], listOfDays))


def parse_prerequisites(course):
    search_str = 'Prereq'

    prereqs_list = [para[para.index(search_str):]
                      for para in course.get('description', [])
                      if search_str in para]

    if not prereqs_list:
        prereqs_list = [note[note.index(search_str):]
                          for note in course.get('notes', [])
                          if search_str in note]

    if prereqs_list:
        return "\n".join(prereqs_list)\
            .replace('Prerequisite: ', '')

    return False


def clean_course(course):
    # Unescape &amp; in course names
    course['name'] = html.unescape(course['coursename'])
    del course['coursename']

    course['section'] = course['coursesection']
    del course['coursesection']

    course['status'] = course['coursestatus']
    del course['coursestatus']

    # Remove <br> tags from the 'notes' field.
    if course['notes']:
        course['notes'] = course['notes'].split('<br>')
        course['notes'] = [' '.join(note.split()) for note in course['notes']]
        course['notes'] = [html.unescape(note) for note in course['notes']]
        course['notes'] = [note for note in course['notes'] if note]

    # Remove the unused varcredits property
    del course['varcredits']

    # Flesh out coursesubtype
    if course['coursesubtype'] and course['coursesubtype'] in course_types:
        course['type'] = course_types[course['coursesubtype']]
    else:
        course['type'] = course['coursesubtype']
        print(f"'{course['type']}' doesn't appear in the types list.")
    del course['coursesubtype']

    # Break apart dept names into lists
    course['departments'] = break_apart_departments(course['deptname'])
    del course['deptname']

    # Turn numbers into numbers
    course['clbid'] = int(course['clbid'])

    if re.search(r'\d{3}$', course['coursenumber']):
        course['number'] = int(course['coursenumber'])
    elif re.match(r'\dXX', course['coursenumber']):
        course['number'] = course['coursenumber']
    elif re.match(r'\d{3}I', course['coursenumber']):
        course['number'] = int(course['coursenumber'][:-1])
    else:
        course['number'] = course['coursenumber']
    del course['coursenumber']

    course['credits'] = float(course['credits'])
    course['crsid'] = int(course['crsid'])
    if course['groupid']:
        course['groupid'] = int(course['groupid'])

    course['enroll'] = int(course['enroll'])
    course['max'] = int(course['max'])

    # Turn booleans into booleans
    course['pn'] = True if course['pn'] is 'Y' else False

{'fraction_non_alphanumeric': 0.14997312309621932, 'fraction_numerical': 0.001612614226841068, 'mean_word_length': 2.6732477788746296, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4402809', 'n_tokens_mistral': 3615, 'n_tokens_neox': 3475, 'n_words': 751}