<commit_before>#! /usr/bin/env python
import mechanize
from BeautifulSoup import BeautifulSoup
import sys
import sqlite3


class Vocab:
    def __init__(self):
        self.word = sys.argv[1] 
        self.def_count = 1
        self.query = {}
        

    def dictionary(self,word):
        br = mechanize.Browser()
        response = br.open('http://www.dictionary.reference.com')
        br.select_form(nr=0)
        br.form['q'] = word 
        br.submit()
        definition = BeautifulSoup(br.response().read())
        trans = definition.findAll('td',{'class':'td3n2'})
        fin = [i.text for i in trans]
        for i in fin: 
            self.query[fin.index(i)] = i
        self.word_database()
        return self.query

    def word_database(self):
        con = sqlite3.connect('/home/oberon/vocab_database/vocab.db')
        with con:
            spot = con.cursor()
            spot.execute("SELECT * FROM Words")
            rows = spot.fetchall() 
            spot.execute("INSERT INTO Words VALUES(?,?)", (len(rows),self.word))
            spot = con.cursor()
            spot.execute("SELECT * FROM Definitions")
            rows_two = spot.fetchall()
            for row in rows_two:
                self.def_count += 1
            for q in self.query:
                spot.execute("INSERT INTO Definitions VALUES(?,?,?)", (self.def_count,self.query[q],len(rows)))
                self.def_count += 1
        
<commit_msg>made classes and cleaned up<commit_after>#! /usr/bin/env python
import mechanize
from BeautifulSoup import BeautifulSoup
import sys
import sqlite3


class Vocab:
    def __init__(self):
        self.word = sys.argv[1] 
        self.def_count = 1
        self.query = {}
        
    def dictionary(self,word):
        br = mechanize.Browser()
        response = br.open('http://www.dictionary.reference.com')
        br.select_form(nr=0)
        br.form['q'] = word 
        br.submit()
        definition = BeautifulSoup(br.response().read())
        trans = definition.findAll('td',{'class':'td3n2'})
        fin = [i.text for i in trans]
        for i in fin: 
            self.query[fin.index(i)] = i
        self.word_database()
        return self.query

    def word_database(self):
        con = sqlite3.connect('/home/oberon/vocab_database/vocab.db')
        with con:
            spot = con.cursor()
            spot.execute("SELECT * FROM Words")
            rows = spot.fetchall() 
            spot.execute("INSERT INTO Words VALUES(?,?)", (len(rows),self.word))
            spot = con.cursor()
            spot.execute("SELECT * FROM Definitions")
            rows_two = spot.fetchall()
            for row in rows_two:
                self.def_count += 1
            for q in self.query:
                spot.execute("INSERT INTO Definitions VALUES(?,?,?)", (self.def_count,self.query[q],len(rows)))
                self.def_count += 1
        

{'fraction_non_alphanumeric': 0.1287435456110155, 'fraction_numerical': 0.006196213425129088, 'mean_word_length': 1.9412955465587045, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3694639', 'n_tokens_mistral': 894, 'n_tokens_neox': 848, 'n_words': 190}