<commit_before>import MySQLdb
import webbrowser
import re
from bs4 import BeautifulSoup as BS

class database(object):
    def __init__(self, hostName="localhost", userName="root",
                 password="", database="crawling"):
        self.db = None
        while self.db is None:
            self.db = self.__getConnection(hostName, userName, password, database)
            
        self.db.autocommit(True)
        self.cur = self.db.cursor()
        
    def __del__(self):
        self.cur.close()
        self.db.close()

    def __getConnection(self, hostName, userName, password, database):
        try:
            return MySQLdb.connect(host = hostName, user = userName,
                                      passwd = password, db = database,
                                      charset='utf8', use_unicode=True)
        except Exception as e:
            print "Init DB connection. Error: %s" %e
            return None

    def getWebsiteHomepage(self, itemID):
        getHomepage_query = '''select Website from item
                                where itemID = %s''' % itemID
        try:
            self.cur.execute(getHomepage_query)    
            url = self.cur.fetchone()
            if url is not None:
                return str(url[0])
            else:
                return "ItemID not found in database"
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.getWebsiteHomepage(itemID)
            
    def getWebsiteFeatureURL(self, itemID):
        query = '''select Feature_URL from item
                                where itemID = %s''' % itemID
        try:
            self.cur.execute(query)
            url = self.cur.fetchone()
            if url is not None:
                return str(url[0])
            else:
                return "ItemID not found in database"
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.getWebsiteFeatureURL(itemID)
            
    # return URL as String
    def getWebsitePriceURL(self, itemID):
        query = '''select Price_URL from item
                                where itemID = %s''' % itemID
        try:
            self.cur.execute(query)
            url = self.cur.fetchone()
            if url is not None:
                if (url[0] <> "Free"):
                    return str(url[0])
                else:
                    return " " #generate a 404 on WBM
            else:
                return "ItemID not found in database"
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.getWebsitePriceURL(itemID)
            
    # return list of itemID
    def getItemID(self, lower, upper):
        query  = '''select itemID from item
                    where itemID >= %s and itemID <= %s''' % (lower, upper)
        try:
            self.cur.execute(query)
            return self.cur.fetchall()
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.getItemID(lower, upper)

    # return True if snapshot had been crawled
    def isSnapshotInDB(self, itemID, index):
        status_query = '''select itemID, url_list_index from snapshot
                        where itemID = %s and
                        url_list_index = %s
                        ''' % (itemID, index)
        try:
            self.cur.execute(status_query)
            status = self.cur.fetchone()
            if status is not None:
                return True
            else:
                return False
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.isSnapshotInDB(itemID, index)
    
    def storeSnapshot(self, itemID, index, date, url, data):
        data = re.escape(data)
        query = """insert ignore into
                snapshot(itemID, url_list_index, snapshot_date,
                            snapshot_url, crawl_data)
                values(%s, %s,
                        STR_TO_DATE(\"%s\", \"%%Y%%m%%d\"), \"%s\", \"%s\");
                """ % (itemID, index, date, url, data)
        try:
            self.cur.execute(query)
        except Exception as e:
            self.__del__()
            self.__init__()
            print '''Storing snapshot itemID = %s: Error (Probably NO url was given?) %s''' % (itemID, e)
            
    # return html data as String
    def retrieveHTML(self, itemID, index):
        query = '''select crawl_data from snapshot
                    where itemID = %s and
                    url_list_index = %s;
                    ''' % (itemID, index)
        try:
            self.cur.execute(query)
            data = self.cur.fetchone()
            if data is not None:
                return str(data[0]) #html as String
            else:
                return "Data has not been crawled"
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.retrieveHTML(itemID, index)
        
    # return itemName as String
    def getItemName(self, itemID):
        query = "select app_name from item where itemID = %s" % itemID
        try:
            self.cur.execute(query)
            itemName = self.cur.fetchone()
            if itemName is not None:
                return str(itemName[0])
            else:
                return "ItemID not found in database"
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.getItemName(itemID)

    # return type: numberOfSnapshots as Int
    def getNumberOfSnapshots(self, itemID):
        query = '''select count(url_list_index) from snapshot
                    where itemID = %s ''' % itemID
        try:
            self.cur.execute(query)
            numberOfSnapshots = self.cur.fetchone()
            if numberOfSnapshots is not None:
                return int(numberOfSnapshots[0])
            else:
                return 0
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.getNumberOfSnapshots(itemID)
<commit_msg>Unicode<commit_after>import MySQLdb
import webbrowser
import re
from bs4 import BeautifulSoup as BS

class database(object):
    def __init__(self, hostName="localhost", userName="root",
                 password="", database="crawling"):
        self.db = None
        while self.db is None:
            self.db = self.__getConnection(hostName, userName, password, database)
            
        self.db.autocommit(True)
        self.cur = self.db.cursor()
        
    def __del__(self):
        self.cur.close()
        self.db.close()

    def __getConnection(self, hostName, userName, password, database):
        try:
            return MySQLdb.connect(host = hostName, user = userName,
                                      passwd = password, db = database,
                                      charset='utf8', use_unicode=True)
        except Exception as e:
            print "Init DB connection. Error: %s" %e
            return None

    def getWebsiteHomepage(self, itemID):
        getHomepage_query = '''select Website from item
                                where itemID = %s''' % itemID
        try:
            self.cur.execute(getHomepage_query)    
            url = self.cur.fetchone()
            if url is not None:
                return str(url[0])
            else:
                return "ItemID not found in database"
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.getWebsiteHomepage(itemID)
            
    def getWebsiteFeatureURL(self, itemID):
        query = '''select Feature_URL from item
                                where itemID = %s''' % itemID
        try:
            self.cur.execute(query)
            url = self.cur.fetchone()
            if url is not None:
                return str(url[0])
            else:
                return "ItemID not found in database"
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.getWebsiteFeatureURL(itemID)
            
    # return URL as String
    def getWebsitePriceURL(self, itemID):
        query = '''select Price_URL from item
                                where itemID = %s''' % itemID
        try:
            self.cur.execute(query)
            url = self.cur.fetchone()
            if url is not None:
                if (url[0] <> "Free"):
                    return str(url[0])
                else:
                    return " " #generate a 404 on WBM
            else:
                return "ItemID not found in database"
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.getWebsitePriceURL(itemID)
            
    # return list of itemID
    def getItemID(self, lower, upper):
        query  = '''select itemID from item
                    where itemID >= %s and itemID <= %s''' % (lower, upper)
        try:
            self.cur.execute(query)
            return self.cur.fetchall()
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.getItemID(lower, upper)

    # return True if snapshot had been crawled
    def isSnapshotInDB(self, itemID, index):
        status_query = '''select itemID, url_list_index from snapshot
                        where itemID = %s and
                        url_list_index = %s
                        ''' % (itemID, index)
        try:
            self.cur.execute(status_query)
            status = self.cur.fetchone()
            if status is not None:
                return True
            else:
                return False
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.isSnapshotInDB(itemID, index)
    
    def storeSnapshot(self, itemID, index, date, url, data):
        try:
            data = re.escape(data.encode('utf-8').decode('utf-8'))
            url = url.decode('unicode')
        except Exception as e:
            print "Cannot store to database. Snapshot URL: %s. Error: %s" % (url, e)
        else:
            query = """insert ignore into
                    snapshot(itemID, url_list_index, snapshot_date,
                                snapshot_url, crawl_data)
                    values(%s, %s,
                            STR_TO_DATE(\"%s\", \"%%Y%%m%%d\"), \"%s\", \"%s\");
                    """ % (itemID, index, date, url, data)
            try:
                self.cur.execute(query)
            except Exception as e:
                self.__del__()
                self.__init__()
                print '''Storing snapshot itemID = %s: Error (Probably NO url was given?) %s''' % (itemID, e)
            
    # return html data as String
    def retrieveHTML(self, itemID, index):
        query = '''select crawl_data from snapshot
                    where itemID = %s and
                    url_list_index = %s;
                    ''' % (itemID, index)
        try:
            self.cur.execute(query)
            data = self.cur.fetchone()
            if data is not None:
                return str(data[0]) #html as String
            else:
                return "Data has not been crawled"
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.retrieveHTML(itemID, index)
        
    # return itemName as String
    def getItemName(self, itemID):
        query = "select app_name from item where itemID = %s" % itemID
        try:
            self.cur.execute(query)
            itemName = self.cur.fetchone()
            if itemName is not None:
                return str(itemName[0])
            else:
                return "ItemID not found in database"
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.getItemName(itemID)

    # return type: numberOfSnapshots as Int
    def getNumberOfSnapshots(self, itemID):
        query = '''select count(url_list_index) from snapshot
                    where itemID = %s ''' % itemID
        try:
            self.cur.execute(query)
            numberOfSnapshots = self.cur.fetchone()
            if numberOfSnapshots is not None:
                return int(numberOfSnapshots[0])
            else:
                return 0
        except (MySQLdb.OperationalError):
            self.__del__()
            self.__init__()
            return self.getNumberOfSnapshots(itemID)

{'fraction_non_alphanumeric': 0.09241724704593474, 'fraction_numerical': 0.0021910947648485796, 'mean_word_length': 1.452033768227168, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7616583', 'n_tokens_mistral': 3429, 'n_tokens_neox': 3318, 'n_words': 915}