<commit_before>from bs4 import BeautifulSoup
import requests
import json
import io
import datetime
import pytz
from calendar import timegm
import re

local_tz = pytz.timezone("America/Los_Angeles")
current_year = datetime.date.today().year
city_council_agendas_url = "https://www.smgov.net/departments/clerk/agendas.aspx"

def get_data():
    with requests.Session() as sess:
        r = sess.get(city_council_agendas_url)
        soup = BeautifulSoup(r.text, 'html.parser')
        agendas = dict()
        table = soup.find('table', {'class': 'agendaTable'})
        rows = table.findAll('tr')
        for row in rows:
            cells = row.findChildren('td')
            date = agenda_date_to_epoch(cells[0])
            if cells[1].string == "Agenda":
                agenda = sess.get(cells[1].findChildren('a', {'href': True})[0]['href']).text
                if "CONSENT CALENDAR" in agenda:
                    agendas[date] = scrape_agenda(agenda, sess)         
    return agendas

def agenda_date_to_epoch(date_str):
    '''Transforms scraped date to epoch time'''
    naive_dt = datetime.datetime.strptime(
        str(current_year) + " " + date_str.string.strip(), '%Y %B %d %I:%M %p')
    local_dt = local_tz.localize(naive_dt, is_dst=None)
    utc_dt = local_dt.astimezone(pytz.utc)
    utc_timetuple = utc_dt.timetuple()
    return timegm(utc_timetuple)

def scrape_agenda(agenda, sess):
    # Searches the entire HTML text for the words since it's not yet parsed into html by BS
    soup_agenda = BeautifulSoup(agenda, 'html.parser')
    tableMeeting = soup_agenda.find('table', {'id': 'MeetingDetail'})
    string_sections = tableMeeting.find_all('strong')<commit_msg>fixed date bug<commit_after>from bs4 import BeautifulSoup
import requests
import json
import io
import datetime
import pytz
from calendar import timegm
import re

local_tz = pytz.timezone("America/Los_Angeles")
current_year = datetime.date.today().year
city_council_agendas_url = "https://www.smgov.net/departments/clerk/agendas.aspx"

def get_data():
    with requests.Session() as sess:
        r = sess.get(city_council_agendas_url)
        soup = BeautifulSoup(r.text, 'html.parser')
        agendas = dict()
        table = soup.find('table', {'class': 'agendaTable'})
        rows = table.findAll('tr')
        for row in rows:
            cells = row.findChildren('td')
            try:
            	date = agenda_date_to_epoch(cells[0])
            except:
            	date = None
            if date and cells[1].string == "Agenda":
                agenda = sess.get(cells[1].findChildren('a', {'href': True})[0]['href']).text
                if "CONSENT CALENDAR" in agenda:
                    agendas[date] = scrape_agenda(agenda, sess)         
    return agendas

def agenda_date_to_epoch(date_str):
    '''Transforms scraped date to epoch time'''
    naive_dt = datetime.datetime.strptime(
        str(current_year) + " " + date_str.string.strip(), '%Y %B %d %I:%M %p')
    local_dt = local_tz.localize(naive_dt, is_dst=None)
    utc_dt = local_dt.astimezone(pytz.utc)
    utc_timetuple = utc_dt.timetuple()
    return timegm(utc_timetuple)

def scrape_agenda(agenda, sess):
    # Searches the entire HTML text for the words since it's not yet parsed into html by BS
    soup_agenda = BeautifulSoup(agenda, 'html.parser')
    tableMeeting = soup_agenda.find('table', {'id': 'MeetingDetail'})
    string_sections = tableMeeting.find_all('strong')
{'fraction_non_alphanumeric': 0.1209046100318933, 'fraction_numerical': 0.0028993911278631487, 'mean_word_length': 3.1767554479418885, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3678442', 'n_tokens_mistral': 1184, 'n_tokens_neox': 1137, 'n_words': 257}