<commit_before>from selenium import webdriver
import os
import re

# tms_url = "http://intranet.andor.com/tms/"
tms_url = '{}/utils/test_tms_pages/TMSClockingsRecordTwoDays.html'.format(os.getcwd())

with webdriver.Chrome(executable_path='{}/utils/chromedriver.exe'.format(os.getcwd())) as browser:
    browser.get(tms_url)

    user_id, user_name = re.match("TMS Clockings for ([0-9]+): (.*)", browser.find_element_by_tag_name("h1").text).groups()
    print("Welcome {} ({})".format(user_name, user_id))

    elem = browser.find_elements_by_class_name("past")
    #for i in range(0, len(elem)):
    #    print("{}: {}".format(i, elem[i].text))

    for i in range(0, int(len(elem)/5)):
        day_str, date_str = re.match("([A-Za-z]+)\n([0-9]{2}/[0-9]{2}/[0-9]{2})", elem[0].text).groups()
        print("{}: {}".format(day_str, date_str))
        print("{}".format(re.findall("([0-9]+:[0-9]+)", elem[1].text)))
        basic_hrs = elem[2].text
        print("Basic Hours: {}".format(basic_hrs))
        auth_absence = elem[3].text
        print("Authorised Absence: {}".format(auth_absence))
        total_hrs = elem[4].text
        print("Total hours: {}".format(total_hrs))

    elem = browser.find_elements_by_class_name("present")
    for i in range(0, int(len(elem)/5)):
        day_str, date_str = re.match("([A-Za-z]+)\n([0-9]{2}/[0-9]{2}/[0-9]{2})", elem[0].text).groups()
        print("{}: {}".format(day_str, date_str))
        print("{}".format(re.findall("([0-9]+:[0-9]+)", elem[1].text)))
        basic_hrs = elem[2].text
        print("Basic Hours: {}".format(basic_hrs))
        auth_absence = elem[3].text
        print("Authorised Absence: {}".format(auth_absence))
        total_hrs = elem[4].text
        print("Total hours: {}".format(total_hrs))
<commit_msg>Added logging to scraper<commit_after>import logging
import logging.config
from selenium import webdriver
import os
import re

logging.config.fileConfig('logging/log_settings.conf')
# tms_url = "http://intranet.andor.com/tms/"
tms_url = '{}/utils/test_tms_pages/TMSClockingsRecordTwoDays.html'.format(os.getcwd())

with webdriver.Chrome(executable_path='{}/utils/chromedriver.exe'.format(os.getcwd())) as browser:
    browser.get(tms_url)

    user_id, user_name = re.match("TMS Clockings for ([0-9]+): (.*)", browser.find_element_by_tag_name("h1").text).groups()
    logging.info("Welcome {} ({})".format(user_name, user_id))

    elem = browser.find_elements_by_class_name("past")
    #for i in range(0, len(elem)):
    #    print("{}: {}".format(i, elem[i].text))

    for i in range(0, int(len(elem)/5)):
        day_str, date_str = re.match("([A-Za-z]+)\n([0-9]{2}/[0-9]{2}/[0-9]{2})", elem[0].text).groups()
        logging.debug("{}: {}".format(day_str, date_str))
        logging.debug("{}".format(re.findall("([0-9]+:[0-9]+)", elem[1].text)))
        basic_hrs = elem[2].text
        logging.debug("Basic Hours: {}".format(basic_hrs))
        auth_absence = elem[3].text
        logging.debug("Authorised Absence: {}".format(auth_absence))
        total_hrs = elem[4].text
        logging.debug("Total hours: {}".format(total_hrs))

    elem = browser.find_elements_by_class_name("present")
    for i in range(0, int(len(elem)/5)):
        day_str, date_str = re.match("([A-Za-z]+)\n([0-9]{2}/[0-9]{2}/[0-9]{2})", elem[0].text).groups()
        logging.debug("{}: {}".format(day_str, date_str))
        logging.debug("{}".format(re.findall("([0-9]+:[0-9]+)", elem[1].text)))
        basic_hrs = elem[2].text
        logging.debug("Basic Hours: {}".format(basic_hrs))
        auth_absence = elem[3].text
        logging.debug("Authorised Absence: {}".format(auth_absence))
        total_hrs = elem[4].text
        logging.debug("Total hours: {}".format(total_hrs))

{'fraction_non_alphanumeric': 0.2161874334398296, 'fraction_numerical': 0.023429179978700747, 'mean_word_length': 3.7738246505717914, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6757417', 'n_tokens_mistral': 1427, 'n_tokens_neox': 1390, 'n_words': 202}