<commit_before>#!/usr/bin/python3

import time, os, errno, threading, queue, logging, traceback, json

import datetime

from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.webdriver.support.ui import Select, WebDriverWait
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.remote.remote_connection import LOGGER

# Disable selenium logging.
LOGGER.setLevel(logging.WARNING)

# URL of myCUinfo.
URL = 'https://portal.prod.cu.edu/psp/epprod/UCB2/ENTP/h/?tab=DEFAULT'
DEFAULT_TIMEOUT = 15

class ScrapeOptions():
    def __init__(self, year, semester, campus, data_path, threads, headless):
        self.year = year
        self.semester = semester
        self.campus = campus
        self.data_path = data_path
        self.threads = threads
        self.headless = headless

class MyCUInfoDriver():
    def __init__(self, username, password, options):
        self.driver = login(username, password, options.headless)
        self.options = options

    def find_id(self, i, timeout=DEFAULT_TIMEOUT):
        return WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.ID, i)))

# Retry action even if exception occurs.
def retry(f, *pargs, max_seconds=30, **kwargs):
    for i in range(max_seconds):
        try:
            return f(*pargs, **kwargs)
        except Exception as e:
            time.sleep(1)
    return f(*pargs, **kwargs)

# Wait for loading icon to vanish.
def wait_for_loading_icon(driver, max_seconds=5):
    SLEEP_INTERVAL = 0.5
    loading_icon = driver.find_elements_by_class_name('ui-icon-loading')[0]
    for i in range(int(max_seconds / SLEEP_INTERVAL)):
        if not loading_icon.is_displayed():
            return
        time.sleep(SLEEP_INTERVAL)
    raise Exception('Loading icon failed to disappear!')


# Log into myCUinfo.
def login(ukeys, pkeys, headless):
    options = webdriver.ChromeOptions()
    if headless:
        options.add_argument('headless')

    driver =  webdriver.Chrome(chrome_options=options)
    driver.get(URL)
    logging.debug('Logging in')

    username = WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'username')))
    username.clear()
    username.send_keys(ukeys)

    passwd = WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'password')))
    passwd.clear()
    passwd.send_keys(pkeys)

    login = WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'submit')))
    login.click()

    return driver


# Enter department info and search.
def runSearch(driver, current, second_time=False):

    institution = 'CUBLD'
    logging.debug('Selecting institution: %s' % institution)
    Select(
        WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'CLASS_SRCH_WRK2_INSTITUTION$31$')))
    ).select_by_value(institution)

    wait_for_loading_icon(driver)

    semester = 'Spring 2018 UC Boulder'
    logging.debug('Selecting semester: %s' % semester)
    Select(
        WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'CLASS_SRCH_WRK2_STRM$35$')))
    ).select_by_visible_text(semester)

    wait_for_loading_icon(driver)

    campus = 'Boulder Main Campus'
    logging.debug('Selecting campus: %s' % campus)
    Select(
        WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'SSR_CLSRCH_WRK_CAMPUS$0')))
    ).select_by_visible_text(campus)

    wait_for_loading_icon(driver)

    logging.debug('Entering department: %s' % current)
    dept_element = WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'SSR_CLSRCH_WRK_SUBJECT$1')))
    # dept_element = retry(driver.find_element_by_id, 'SSR_CLSRCH_WRK_SUBJECT$1')
    dept_element.clear()
    dept_element.send_keys(current)

    # Chem has too many classes! AAAAAH
    WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'SSR_CLSRCH_WRK_CATALOG_NBR$2'))).clear()
    # retry(driver.find_element_by_id, 'SSR_CLSRCH_WRK_CATALOG_NBR$2').clear()
    if current == 'CHEM':
        logging.debug('Splitting CHEM into two groups')

        course_code_text = ('greater' if second_time else 'less') + ' than or equal to'
        Select(
            retry(driver.find_element_by_id, 'SSR_CLSRCH_WRK_SSR_EXACT_MATCH1$2')
        ).select_by_visible_text(course_code_text)<commit_msg>now it will scrape the right semester!<commit_after>#!/usr/bin/python3

import time, os, errno, threading, queue, logging, traceback, json

import datetime

from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.webdriver.support.ui import Select, WebDriverWait
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.remote.remote_connection import LOGGER

# Disable selenium logging.
LOGGER.setLevel(logging.WARNING)

# URL of myCUinfo.
URL = 'https://portal.prod.cu.edu/psp/epprod/UCB2/ENTP/h/?tab=DEFAULT'
DEFAULT_TIMEOUT = 15

class ScrapeOptions():
    def __init__(self, year, semester, campus, data_path, threads, headless):
        self.year = year
        self.semester = semester
        self.campus = campus
        self.data_path = data_path
        self.threads = threads
        self.headless = headless

class MyCUInfoDriver():
    def __init__(self, username, password, options):
        self.driver = login(username, password, options.headless)
        self.options = options

    def find_id(self, i, timeout=DEFAULT_TIMEOUT):
        return WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.ID, i)))

# Retry action even if exception occurs.
def retry(f, *pargs, max_seconds=30, **kwargs):
    for i in range(max_seconds):
        try:
            return f(*pargs, **kwargs)
        except Exception as e:
            time.sleep(1)
    return f(*pargs, **kwargs)

# Wait for loading icon to vanish.
def wait_for_loading_icon(driver, max_seconds=5):
    SLEEP_INTERVAL = 0.5
    loading_icon = driver.find_elements_by_class_name('ui-icon-loading')[0]
    for i in range(int(max_seconds / SLEEP_INTERVAL)):
        if not loading_icon.is_displayed():
            return
        time.sleep(SLEEP_INTERVAL)
    raise Exception('Loading icon failed to disappear!')


# Log into myCUinfo.
def login(ukeys, pkeys, headless):
    options = webdriver.ChromeOptions()
    if headless:
        options.add_argument('headless')

    driver =  webdriver.Chrome(chrome_options=options)
    driver.get(URL)
    logging.debug('Logging in')

    username = WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'username')))
    username.clear()
    username.send_keys(ukeys)

    passwd = WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'password')))
    passwd.clear()
    passwd.send_keys(pkeys)

    login = WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'submit')))
    login.click()

    return driver


# Enter department info and search.
def runSearch(driver, current, second_time=False):

    institution = 'CUBLD'
    logging.debug('Selecting institution: %s' % institution)
    Select(
        WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'CLASS_SRCH_WRK2_INSTITUTION$31$')))
    ).select_by_value(institution)

    wait_for_loading_icon(driver)

    semester = 'Fall 2018 UC Boulder'
    logging.debug('Selecting semester: %s' % semester)
    Select(
        WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'CLASS_SRCH_WRK2_STRM$35$')))
    ).select_by_visible_text(semester)

    wait_for_loading_icon(driver)

    campus = 'Boulder Main Campus'
    logging.debug('Selecting campus: %s' % campus)
    Select(
        WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'SSR_CLSRCH_WRK_CAMPUS$0')))
    ).select_by_visible_text(campus)

    wait_for_loading_icon(driver)

    logging.debug('Entering department: %s' % current)
    dept_element = WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'SSR_CLSRCH_WRK_SUBJECT$1')))
    # dept_element = retry(driver.find_element_by_id, 'SSR_CLSRCH_WRK_SUBJECT$1')
    dept_element.clear()
    dept_element.send_keys(current)

    # Chem has too many classes! AAAAAH
    WebDriverWait(driver, DEFAULT_TIMEOUT).until(EC.presence_of_element_located((By.ID, 'SSR_CLSRCH_WRK_CATALOG_NBR$2'))).clear()
    # retry(driver.find_element_by_id, 'SSR_CLSRCH_WRK_CATALOG_NBR$2').clear()
    if current == 'CHEM':
        logging.debug('Splitting CHEM into two groups')

        course_code_text = ('greater' if second_time else 'less') + ' than or equal to'
        Select(
            retry(driver.find_element_by_id, 'SSR_CLSRCH_WRK_SSR_EXACT_MATCH1$2')
        ).select_by_visible_text(course_code_text)
{'fraction_non_alphanumeric': 0.10831070070613796, 'fraction_numerical': 0.006083650190114068, 'mean_word_length': 4.29384703852789, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4476998', 'n_tokens_mistral': 3130, 'n_tokens_neox': 3004, 'n_words': 577}