<commit_before>from __future__ import (
    absolute_import,
    print_function,
)

import re

from bs4 import BeautifulSoup
import requests

from constants import CURRENT_SEM
from contact import send_alerts
from dbhelper import (
    db_session,
    init_db,
)
from models import (
    Alert,
    Course,
    Dept,
    Klass,
)
from util import (
    hour_of_day_to_seconds_since_midnight,
    web_day_to_int_day,
    web_status_to_db_status,
)

DEBUG_PRINT = False

init_db()


def scrape_course_and_classes(course_id, dept_id, name, klasses):
    '''scrape all the classes in a course'''
    # add the course to the db
    course = Course(course_id=course_id, dept_id=dept_id, name=name)
    db_session.merge(course)

    klasses_to_delete = {klass.klass_id: klass for klass in db_session.query(Klass)
                         .filter_by(course_id=course_id, dept_id=dept_id)
                         .all()}

    for row in klasses:
        klass_type, _, klass_id, _, status, enrollment, _, time_and_place = (d.get_text() for d in row)
        status = web_status_to_db_status(status)
        klass_id = int(klass_id)
        m = re.search(r'(\d+)/(\d+).*', enrollment)
        enrolled = int(m.group(1))
        capacity = int(m.group(2))
        # separate the time from the place

        if DEBUG_PRINT:
            print('Storing class with class id:', klass_id)
            print('Class row fields:', [d.get_text() for d in row])

        m = re.search(r'(\w+) +(\d+(?::\d+)?(?:-\d+(?::\d+)?)?) *#? *(?: *\((?:.*, *)*(.*?)\))?',
                      time_and_place) if time_and_place else None
        if m:<commit_msg>don't automatically init db in scraper.py<commit_after>from __future__ import (
    absolute_import,
    print_function,
)

import re

from bs4 import BeautifulSoup
import requests

from constants import CURRENT_SEM
from contact import send_alerts
from dbhelper import db_session
from models import (
    Alert,
    Course,
    Dept,
    Klass,
)
from util import (
    hour_of_day_to_seconds_since_midnight,
    web_day_to_int_day,
    web_status_to_db_status,
)

DEBUG_PRINT = False


def scrape_course_and_classes(course_id, dept_id, name, klasses):
    '''scrape all the classes in a course'''
    # add the course to the db
    course = Course(course_id=course_id, dept_id=dept_id, name=name)
    db_session.merge(course)

    klasses_to_delete = {klass.klass_id: klass for klass in db_session.query(Klass)
                         .filter_by(course_id=course_id, dept_id=dept_id)
                         .all()}

    for row in klasses:
        klass_type, _, klass_id, _, status, enrollment, _, time_and_place = (d.get_text() for d in row)
        status = web_status_to_db_status(status)
        klass_id = int(klass_id)
        m = re.search(r'(\d+)/(\d+).*', enrollment)
        enrolled = int(m.group(1))
        capacity = int(m.group(2))
        # separate the time from the place

        if DEBUG_PRINT:
            print('Storing class with class id:', klass_id)
            print('Class row fields:', [d.get_text() for d in row])

        m = re.search(r'(\w+) +(\d+(?::\d+)?(?:-\d+(?::\d+)?)?) *#? *(?: *\((?:.*, *)*(.*?)\))?',
                      time_and_place) if time_and_place else None
        if m:
{'fraction_non_alphanumeric': 0.12404052809333743, 'fraction_numerical': 0.00184218606079214, 'mean_word_length': 2.7448275862068967, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1812082', 'n_tokens_mistral': 1139, 'n_tokens_neox': 1115, 'n_words': 266}