<commit_before>#!/usr/bin/env python2

"""
Usage:
    ./scraper.py [options]

Options:
    -h, --help                      Prints this help message
    -o FILE, --output FILE          Specifies output file
    -c COLLEGE, --college COLLEGE   Specifies a specific college
    -l, --last-term-only            Only ouputs the last term
    -u URL, --url URL               Specify an alternate class-search url
    -v, --verbose                   Turns on verbose logging
"""

import datetime
import itertools
import json
import logging
import time

from collections import OrderedDict
from multiprocessing import Pool, cpu_count
from os import path

import requests

from BeautifulSoup import BeautifulSoup
from docopt import docopt

BASE_URL = "http://www.unomaha.edu/registrar/students/before-you-enroll/class-search/"

terms = [1158]

def get_college_data((college, term)):
    """Returns a dictionary containing all classes within college and term"""
    logging.info("Processing college {}".format(college))

    time.sleep(1)
    page = requests.get("{}?term={}&session=&subject={}&catalog_nbr=&career=&instructor=&class_start_time=&class_end_time=&location=&special=&instruction_mode=".format(BASE_URL, term, college))
    soup = BeautifulSoup(page.text)

    if len(soup.findAll("div", {'class': 'dotted-bottom'})) == 0:
        logging.error("No classes for college {}, term {}".format(college, term))

    classes = OrderedDict()
    #loop through each class in the college
    for dotted in soup.findAll("div", {'class': 'dotted-bottom'}):
        cls = OrderedDict()

        number = dotted.find("h2")
        if number:
            class_number = number.text.split(" ")[-1]
        else:
            class_number = "-"

        title = dotted.find("p")
        if title:
            cls['title'] = title.text
        else:
            cls['title'] = "-"
<commit_msg>update url to use https<commit_after>#!/usr/bin/env python2

"""
Usage:
    ./scraper.py [options]

Options:
    -h, --help                      Prints this help message
    -o FILE, --output FILE          Specifies output file
    -c COLLEGE, --college COLLEGE   Specifies a specific college
    -l, --last-term-only            Only ouputs the last term
    -u URL, --url URL               Specify an alternate class-search url
    -v, --verbose                   Turns on verbose logging
"""

import datetime
import itertools
import json
import logging
import time

from collections import OrderedDict
from multiprocessing import Pool, cpu_count
from os import path

import requests

from BeautifulSoup import BeautifulSoup
from docopt import docopt

BASE_URL = "https://www.unomaha.edu/registrar/students/before-you-enroll/class-search/"

terms = [1158]

def get_college_data((college, term)):
    """Returns a dictionary containing all classes within college and term"""
    logging.info("Processing college {}".format(college))

    time.sleep(1)
    page = requests.get("{}?term={}&session=&subject={}&catalog_nbr=&career=&instructor=&class_start_time=&class_end_time=&location=&special=&instruction_mode=".format(BASE_URL, term, college))
    soup = BeautifulSoup(page.text)

    if len(soup.findAll("div", {'class': 'dotted-bottom'})) == 0:
        logging.error("No classes for college {}, term {}".format(college, term))

    classes = OrderedDict()
    #loop through each class in the college
    for dotted in soup.findAll("div", {'class': 'dotted-bottom'}):
        cls = OrderedDict()

        number = dotted.find("h2")
        if number:
            class_number = number.text.split(" ")[-1]
        else:
            class_number = "-"

        title = dotted.find("p")
        if title:
            cls['title'] = title.text
        else:
            cls['title'] = "-"

{'fraction_non_alphanumeric': 0.1248339973439575, 'fraction_numerical': 0.004780876494023904, 'mean_word_length': 2.76976976976977, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '740308', 'n_tokens_mistral': 1147, 'n_tokens_neox': 1094, 'n_words': 314}