<commit_before>import logging
import os

import pandas as pd
import requests
from bs4 import BeautifulSoup
from selenium import webdriver

logging.basicConfig(filename='scrape.log', level=20)
logging.info('Initialized logger')


def get_soup_object(html):
    return BeautifulSoup(html)


def get_dynamic_content(link):
    try:
        driver = webdriver.PhantomJS()
        logging.info("Webdriver initiated {0}".format(driver))
    except Exception as exp:
        logging.fatal("Webdriver exception: {exception}".format(exception=exp))
        raise exp

    driver.get(link)
    r = requests.get("url")
    if r.status_code != 200:
        raise Exception("Status code is not 200. Exiting!")
    html = driver.page_source
    soup = get_soup_object(html)
    # Dump the page source to log for debugging purpose, this will work only if level is set to 10 in line number 5
    logging.debug(soup.prettify())
    parsedata(soup)


def parsedata(soup):
    victims = soup.find_all("div", {"class": "homicide"})
    for victim in victims:
        # Set default values as NA for each victim
        date = name = age = race = cause = neighbourhood = murder_time = addr = "NA"
        try:
            date = victim.find("div", {"class": "date"}).get_text().strip()
        except Exception as exp:
            pass<commit_msg>Oops, use the correct link<commit_after>import logging
import os

import pandas as pd
import requests
from bs4 import BeautifulSoup
from selenium import webdriver

logging.basicConfig(filename='scrape.log', level=20)
logging.info('Initialized logger')


def get_soup_object(html):
    return BeautifulSoup(html)


def get_dynamic_content(link):
    try:
        driver = webdriver.PhantomJS()
        logging.info("Webdriver initiated {0}".format(driver))
    except Exception as exp:
        logging.fatal("Webdriver exception: {exception}".format(exception=exp))
        raise exp

    driver.get(link)
    r = requests.get(link)
    if r.status_code != 200:
        raise Exception("Status code is not 200. Exiting!")
    html = driver.page_source
    soup = get_soup_object(html)
    # Dump the page source to log for debugging purpose, this will work only if level is set to 10 in line number 5
    logging.debug(soup.prettify())
    parsedata(soup)


def parsedata(soup):
    victims = soup.find_all("div", {"class": "homicide"})
    for victim in victims:
        # Set default values as NA for each victim
        date = name = age = race = cause = neighbourhood = murder_time = addr = "NA"
        try:
            date = victim.find("div", {"class": "date"}).get_text().strip()
        except Exception as exp:
            pass
{'fraction_non_alphanumeric': 0.10330578512396695, 'fraction_numerical': 0.009767092411720512, 'mean_word_length': 3.20695102685624, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '948785', 'n_tokens_mistral': 813, 'n_tokens_neox': 770, 'n_words': 247}