<commit_before>#imports
import sqlite3
import re
from urllib.request import urlopen as uReq
from bs4 import BeautifulSoup as soup

#DECLARE ALL SQL FUNCTIONS:
conn = sqlite3.connect('chucksOnEbay.db')
c = conn.cursor()

def create_table():
	c.execute("CREATE TABLE IF NOT EXISTS chucks(id TEXT, list_title TEXT, date_sold TEXT, price REAL)")

def data_entry():
	c.execute("INSERT INTO chucks (id, list_title, date_sold, price) VALUES (?, ?, ?, ?)",
		(item_id, item_title, item_date, item_price))
	conn.commit()

def no_duplicates(x):
	c.execute("SELECT id FROM chucks WHERE id = ?", (x,))
	data = c.fetchone()
	if data is None:
		return 0
	else:
		return 1

def date_format(date):
	months = {"Jan" : '01', "Feb" : '02', "Mar" : '03', "Apr" : '04', "May" : '05', "Jun" : '06', "Jul" : '07', "Aug" : '08', "Sep" : '09', "Oct" : '10', "Nov" : '11', "Dec" : '12'}
	month = str(date[:3])
	day = str(date[4:6])
	month_num = months[month]
	return month_num + "/" + day

my_url = 'https://www.ebay.com/sch/i.html?_from=R40&_sacat=0&LH_Complete=1&LH_Sold=1&LH_ItemCondition=1000&_nkw=converse%20chuck%20taylor%202&_dcat=15709&US%2520Shoe%2520Size%2520%2528Men%2527s%2529=11&rt=nc&_trksid=p2045573.m1684'

#Opening connection and grabbing the page
uClient = uReq(my_url)
#make var before .read() so you don't lose the data
page_html = uClient.read()
#close the client because it is an open connection
uClient.close()

# html parsing
page_soup = soup(page_html, "html.parser")

#class nllclt is only there when there are 0 results
if bool(page_soup.find("span", {"class": "nllclt"})) == True
	return 0

#find the first of this only because Ebay sometimes adds suggested results that don't match right away
matches = page_soup.find("ul", {"class": "gv-ic"})

# grabs each sale
containers = matches.findAll("li",{"class":"sresult"})

# Create table, comment out after making it the first time
create_table()

for container in containers:
	#extract unique identifier
	#all ints in python 3 are long longs
	item_id = container.get("id")

	#if item_id exists, loop to next entry
	if no_duplicates(item_id) == 1:
		continue

	# extract item title from html
	item_title = container.h3.a.text
	
	# extract date and time sold from html
	date_container = container.find("span", {"class":"lcol"})
	date = date_container.text[:6]
	item_date = date_format(date).strip()

	# extract price
	price_container = container.find("span", {"class":"bidsold"}) 
	# strip data to clean up and fix formatting
	item_price = price_container.text.replace("$", "").strip()
	#if has "\xa0to " get the average
	if bool(re.search("\xa0to ", item_price)) == False:
		item_price = item_price
	elif bool(re.search("\xa0to ", item_price)) == True:
		arr = item_price.split("\xa0to ")
		p1 = float(arr[0])
		p2 = float(arr[1])
		item_price = (p1 + p2) / 2.00

	#write outputs in csv file and format accordingly
	data_entry()

# always close csv file when done writing it
c.close()
conn.close()

#print output in terminal to know that it went all the way through
print("check the database!")
<commit_msg>FEAT: Add shoe sizes array and size column to db<commit_after>#imports
import sqlite3
import re
from urllib.request import urlopen as uReq
from bs4 import BeautifulSoup as soup

#DECLARE SQL:
conn = sqlite3.connect('chucksOnEbay.db')
c = conn.cursor()

def create_table():
	c.execute("CREATE TABLE IF NOT EXISTS chucks(id TEXT, list_title TEXT, shoe_size TEXT, date_sold TEXT, price REAL)")

def data_entry():
	c.execute("INSERT INTO chucks (id, list_title, shoe_size, date_sold, price) VALUES (?, ?, ?, ?, ?)",
		(item_id, item_title, size, item_date, item_price))
	conn.commit()

def no_duplicates(x):
	c.execute("SELECT id FROM chucks WHERE id = ?", (x,))
	data = c.fetchone()
	if data is None:
		return 0
	else:
		return 1

def date_format(date):
	months = {"Jan" : '01', "Feb" : '02', "Mar" : '03', "Apr" : '04', "May" : '05', "Jun" : '06', "Jul" : '07', "Aug" : '08', "Sep" : '09', "Oct" : '10', "Nov" : '11', "Dec" : '12'}
	month = str(date[:3])
	day = str(date[4:6])
	month_num = months[month]
	return month_num + "/" + day

sizes = ['5','5%252E5','6','6%252E5','7','7%252E5','8','8%252E5','9','9%252E5','10','10%252E5','11','11%252E5','12','12%252E5','13','13%252E5','14','14%252E5']

size = sizes[11]

my_url = 'https://www.ebay.com/sch/i.html?_from=R40&_sacat=0&LH_Complete=1&LH_Sold=1&LH_ItemCondition=1000&_nkw=converse%20chuck%20taylor%202&_dcat=15709&US%2520Shoe%2520Size%2520%2528Men%2527s%2529=' + sizes[11] + '&rt=nc&_trksid=p2045573.m1684'

#Opening connection and grabbing the page
uClient = uReq(my_url)
#make var before .read() so you don't lose the data
page_html = uClient.read()
#close the client because it is an open connection
uClient.close()

# html parsing
page_soup = soup(page_html, "html.parser")

#class nllclt is only there when there are 0 results
#if bool(page_soup.find("span", {"class": "nllclt"})) == True:
	

#find the first of this only because Ebay sometimes adds suggested results that don't match right away
matches = page_soup.find("ul", {"class": "gv-ic"})

# grabs each sale
containers = matches.findAll("li",{"class":"sresult"})

# Create table, comment out after making it the first time
create_table()

for container in containers:
	#extract unique identifier
	#all ints in python 3 are long longs
	item_id = container.get("id")

	#if item_id exists, loop to next entry
	if no_duplicates(item_id) == 1:
		continue

	# extract item title from html
	item_title = container.h3.a.text
	
	# extract date and time sold from html
	date_container = container.find("span", {"class":"lcol"})
	date = date_container.text[:6]
	item_date = date_format(date).strip()

	# extract price
	price_container = container.find("span", {"class":"bidsold"}) 
	# strip data to clean up and fix formatting
	item_price = price_container.text.replace("$", "").strip()
	#if has "\xa0to " get the average
	if bool(re.search("\xa0to ", item_price)) == False:
		item_price = item_price
	elif bool(re.search("\xa0to ", item_price)) == True:
		arr = item_price.split("\xa0to ")
		p1 = float(arr[0])
		p2 = float(arr[1])
		item_price = (p1 + p2) / 2.00

	#write outputs in csv file and format accordingly
	data_entry()

# always close db connection when done writing it
c.close()
conn.close()

#print output in terminal to know that it went all the way through
print("check the database!")

{'fraction_non_alphanumeric': 0.15356806035837786, 'fraction_numerical': 0.04542596667714555, 'mean_word_length': 5.06, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2796490', 'n_tokens_mistral': 2534, 'n_tokens_neox': 2301, 'n_words': 671}