<commit_before>"""Database Toolkit Data Model
This module contains basic class definitions for the Database Toolkit platform.

 Database: Represents a database to be created.
    Table: Represents an individual table in a Database.
   Engine: Contains the Database and Table objects and database platform-
           specific methods to manipulate databases.
"""

import sys
import os
import getpass
import getopt
import zipfile
import urllib

RAW_DATA_LOCATION = "raw_data"
    
class Cleanup:
    """This class represents a custom cleanup function and a dictionary of 
    arguments to be passed to that function."""
    def __init__(self, function, args):
        self.function = function
        self.args = args

def no_cleanup(value, args):
    """Default cleanup function, returns the unchanged value."""
    return value        

class Database:
    """Information about a database."""
    dbname = ""
    drop = True
    opts = dict()
    
class Table:
    """Information about a database table."""
    tablename = ""
    pk = True
    hasindex = False
    record_id = 0
    delimiter = None
    columns = []
    drop = True
    header_rows = 1
    fixedwidth = False
    def __init__(self):        
        self.cleanup = Cleanup(no_cleanup, None)        
    
class Engine():
    """A generic database system. Specific database platforms will inherit 
    from this class."""
    name = ""
    db = None
    table = None
    connection = None
    cursor = None
    keep_raw_data = True
    use_local = True
    datatypes = []
    required_opts = []
    pkformat = "%s PRIMARY KEY"
    script = None    
    def add_to_table(self):
        """This function adds data to a table from one or more lines specified 
        in engine.table.source."""        
        for line in self.table.source:
            
            line = line.strip()
            if line:
                self.table.record_id += 1            
                linevalues = []
                if (self.table.pk and self.table.hasindex == False):
                    column = 0
                else:
                    column = -1
                 
                for value in self.extract_values(line):
                    column += 1
                    thiscolumn = self.table.columns[column][1][0]
                    # If data type is "skip" ignore the value
                    if thiscolumn == "skip":
                        pass
                    elif thiscolumn == "combine":
                        # If "combine" append value to end of previous column
                        linevalues[len(linevalues) - 1] += " " + value 
                    else:
                        # Otherwise, add new value
                        linevalues.append(value) 
                            
                # Build insert statement with the correct # of values                
                cleanvalues = [self.format_insert_value(self.table. \
                                                        cleanup.function
                                                        (value, 
                                                         self.table. \
                                                         cleanup.args)) 
                               for value in linevalues]
                insertstatement = self.insert_statement(cleanvalues)
                prompt = "Inserting rows: "
                sys.stdout.write(prompt + str(self.table.record_id) + "\b" *
                                 (len(str(self.table.record_id)) + 
                                  len(prompt)))
                self.cursor.execute(insertstatement)
                
        print "\n Done!"
        self.connection.commit()        
    def convert_data_type(self, datatype):
        """Converts DBTK generic data types to database platform specific data
        types"""
        datatypes = dict()
        thistype = datatype[0]
        thispk = False
        if thistype[0:3] == "pk-":
            thistype = thistype.lstrip("pk-")
            thispk = True
        customtypes = ("auto", "int", "double", "char", "bit")
        for i in range(0, len(customtypes)):
            datatypes[customtypes[i]] = i
        datatypes["combine"], datatypes["skip"] = [-1, -1]        
        mydatatypes = self.datatypes
        thisvartype = datatypes[thistype]
        if thisvartype > -1:
            type = mydatatypes[thisvartype]
            if len(datatype) > 1:
                type += "(" + str(datatype[1]) + ")"
        else:
            type = ""
        if thispk:
            type = self.pkformat % type
        return type    
    def create_db(self):
        """Creates a new database based on settings supplied in Database object
        engine.db"""
        print "Creating database " + self.db.dbname + " . . ."
        # Create the database    
        self.cursor.execute(self.create_db_statement())
    def create_db_statement(self):
        """Returns a SQL statement to create a database."""
        if self.db.drop:
            self.cursor.execute(self.drop_statement("DATABASE", 
                                                    self.db.dbname))
            createstatement = "CREATE DATABASE " + self.db.dbname
        else:
            createstatement = "CREATE DATABASE IF NOT EXISTS " + db.dbname
        return createstatement
    def create_raw_data_dir(self):
        """Checks to see if the archive directory exists and creates it if 
        necessary."""
        if not os.path.exists(RAW_DATA_LOCATION):
            os.makedirs(RAW_DATA_LOCATION)
        path = os.path.join(RAW_DATA_LOCATION, self.script.shortname)
        if not os.path.exists(path):
            os.makedirs(path)            
    def create_table(self):
        """Creates a new database table based on settings supplied in Table 
        object engine.table."""
        print "Creating table " + self.table.tablename + ". . ."
        createstatement = self.create_table_statement()
        self.cursor.execute(createstatement)
    def create_table_statement(self):
        """Returns a SQL statement to create a table."""
        if self.table.drop:
            self.cursor.execute(self.drop_statement("TABLE", self.tablename()))
            createstatement = "CREATE TABLE " + self.tablename() + " ("
        else:
            createstatement = ("CREATE TABLE IF NOT EXISTS " + 
                               self.tablename() + " (")    
        for item in self.table.columns:
            if (item[1][0] != "skip") and (item[1][0] != "combine"):
                createstatement += (item[0] + " "
                                    + self.convert_data_type(item[1]) + ", ")    

        createstatement = createstatement.rstrip(', ')    
        createstatement += " );"
        return createstatement
    def download_file(self, url, filename):
        file = urllib.urlopen(url) 
        local_file = open(self.format_filename(filename), 'wb')
        local_file.write(file.read())
        local_file.close()
        file.close()
    def download_files_from_archive(self, url, filenames):
        """Downloads one or more files from an archive into the raw data
        directory."""
        downloaded = False
        archivename = self.format_filename(url.split('/')[-1])
        
        for filename in filenames:
            if self.use_local and os.path.isfile(self.format_filename(filename)):
                # Use local copy
                print "Using local copy of " + filename
                self.insert_data_from_file(self.format_filename(filename))
            else:
                self.create_raw_data_dir()
                
                if not downloaded:                    
                    self.download_file(url, url.split('/')[-1])
                    downloaded = True     
                        
                local_zip = zipfile.ZipFile(archivename)
                fileloc = self.format_filename(filename)
                        
                open_zip = local_zip.open(filename)
                unzipped_file = open(fileloc, 'wb')
                unzipped_file.write(open_zip.read())
                unzipped_file.close()
                open_zip.close()
                
                local_zip.close()                                            
        try:
            os.remove(archivename)
        except:
            pass            
    def drop_statement(self, objecttype, objectname):
        """Returns a drop table or database SQL statement."""
        dropstatement = "DROP %s IF EXISTS %s" % (objecttype, objectname)
        return dropstatement<commit_msg>Added dbtks_gentry.py. Removed the deletion of the archive after download_files_from_archive, since keeping a copy of the archive can be useful, i.e. to get a list of the files it contains.<commit_after>"""Database Toolkit Data Model
This module contains basic class definitions for the Database Toolkit platform.

 Database: Represents a database to be created.
    Table: Represents an individual table in a Database.
   Engine: Contains the Database and Table objects and database platform-
           specific methods to manipulate databases.
"""

import sys
import os
import getpass
import getopt
import zipfile
import urllib

RAW_DATA_LOCATION = "raw_data"
    
class Cleanup:
    """This class represents a custom cleanup function and a dictionary of 
    arguments to be passed to that function."""
    def __init__(self, function, args):
        self.function = function
        self.args = args

def no_cleanup(value, args):
    """Default cleanup function, returns the unchanged value."""
    return value        

class Database:
    """Information about a database."""
    dbname = ""
    drop = True
    opts = dict()
    
class Table:
    """Information about a database table."""
    tablename = ""
    pk = True
    hasindex = False
    record_id = 0
    delimiter = None
    columns = []
    drop = True
    header_rows = 1
    fixedwidth = False
    def __init__(self):        
        self.cleanup = Cleanup(no_cleanup, None)        
    
class Engine():
    """A generic database system. Specific database platforms will inherit 
    from this class."""
    name = ""
    db = None
    table = None
    connection = None
    cursor = None
    keep_raw_data = True
    use_local = True
    datatypes = []
    required_opts = []
    pkformat = "%s PRIMARY KEY"
    script = None    
    def add_to_table(self):
        """This function adds data to a table from one or more lines specified 
        in engine.table.source."""        
        for line in self.table.source:
            
            line = line.strip()
            if line:
                self.table.record_id += 1            
                linevalues = []
                if (self.table.pk and self.table.hasindex == False):
                    column = 0
                else:
                    column = -1
                 
                for value in self.extract_values(line):
                    column += 1
                    thiscolumn = self.table.columns[column][1][0]
                    # If data type is "skip" ignore the value
                    if thiscolumn == "skip":
                        pass
                    elif thiscolumn == "combine":
                        # If "combine" append value to end of previous column
                        linevalues[len(linevalues) - 1] += " " + value 
                    else:
                        # Otherwise, add new value
                        linevalues.append(value) 
                            
                # Build insert statement with the correct # of values                
                cleanvalues = [self.format_insert_value(self.table. \
                                                        cleanup.function
                                                        (value, 
                                                         self.table. \
                                                         cleanup.args)) 
                               for value in linevalues]
                insertstatement = self.insert_statement(cleanvalues)
                prompt = "Inserting rows: "
                sys.stdout.write(prompt + str(self.table.record_id) + "\b" *
                                 (len(str(self.table.record_id)) + 
                                  len(prompt)))
                self.cursor.execute(insertstatement)
                
        print "\n Done!"
        self.connection.commit()        
    def convert_data_type(self, datatype):
        """Converts DBTK generic data types to database platform specific data
        types"""
        datatypes = dict()
        thistype = datatype[0]
        thispk = False
        if thistype[0:3] == "pk-":
            thistype = thistype.lstrip("pk-")
            thispk = True
        customtypes = ("auto", "int", "double", "char", "bit")
        for i in range(0, len(customtypes)):
            datatypes[customtypes[i]] = i
        datatypes["combine"], datatypes["skip"] = [-1, -1]        
        mydatatypes = self.datatypes
        thisvartype = datatypes[thistype]
        if thisvartype > -1:
            type = mydatatypes[thisvartype]
            if len(datatype) > 1:
                type += "(" + str(datatype[1]) + ")"
        else:
            type = ""
        if thispk:
            type = self.pkformat % type
        return type    
    def create_db(self):
        """Creates a new database based on settings supplied in Database object
        engine.db"""
        print "Creating database " + self.db.dbname + " . . ."
        # Create the database    
        self.cursor.execute(self.create_db_statement())
    def create_db_statement(self):
        """Returns a SQL statement to create a database."""
        if self.db.drop:
            self.cursor.execute(self.drop_statement("DATABASE", 
                                                    self.db.dbname))
            createstatement = "CREATE DATABASE " + self.db.dbname
        else:
            createstatement = "CREATE DATABASE IF NOT EXISTS " + db.dbname
        return createstatement
    def create_raw_data_dir(self):
        """Checks to see if the archive directory exists and creates it if 
        necessary."""
        if not os.path.exists(RAW_DATA_LOCATION):
            os.makedirs(RAW_DATA_LOCATION)
        path = os.path.join(RAW_DATA_LOCATION, self.script.shortname)
        if not os.path.exists(path):
            os.makedirs(path)            
    def create_table(self):
        """Creates a new database table based on settings supplied in Table 
        object engine.table."""
        print "Creating table " + self.table.tablename + ". . ."
        createstatement = self.create_table_statement()
        self.cursor.execute(createstatement)
    def create_table_statement(self):
        """Returns a SQL statement to create a table."""
        if self.table.drop:
            self.cursor.execute(self.drop_statement("TABLE", self.tablename()))
            createstatement = "CREATE TABLE " + self.tablename() + " ("
        else:
            createstatement = ("CREATE TABLE IF NOT EXISTS " + 
                               self.tablename() + " (")    
        for item in self.table.columns:
            if (item[1][0] != "skip") and (item[1][0] != "combine"):
                createstatement += (item[0] + " "
                                    + self.convert_data_type(item[1]) + ", ")    

        createstatement = createstatement.rstrip(', ')    
        createstatement += " );"
        return createstatement
    def download_file(self, url, filename):
        self.create_raw_data_dir()
        if not self.use_local or not os.path.isfile(self.format_filename(filename)):
            print "Downloading " + filename + " . . ."
            file = urllib.urlopen(url) 
            local_file = open(self.format_filename(filename), 'wb')
            local_file.write(file.read())
            local_file.close()
            file.close()
    def download_files_from_archive(self, url, filenames):
        """Downloads one or more files from an archive into the raw data
        directory."""
        downloaded = False
        archivename = self.format_filename(url.split('/')[-1])
        
        for filename in filenames:
            if self.use_local and os.path.isfile(self.format_filename(filename)):
                # Use local copy
                print "Using local copy of " + filename
                self.insert_data_from_file(self.format_filename(filename))
            else:
                self.create_raw_data_dir()
                
                if not downloaded:
                    self.download_file(url, url.split('/')[-1])
                    downloaded = True     
                        
                local_zip = zipfile.ZipFile(archivename)
                fileloc = self.format_filename(os.path.basename(filename))
                        
                open_zip = local_zip.open(filename)
                unzipped_file = open(fileloc, 'wb')
                unzipped_file.write(open_zip.read())
                unzipped_file.close()
                open_zip.close()
                
                local_zip.close()                                            
        if not self.keep_raw_data:
            try:
                os.remove(archivename)
            except:
                pass            
    def drop_statement(self, objecttype, objectname):
        """Returns a drop table or database SQL statement."""
        dropstatement = "DROP %s IF EXISTS %s" % (objecttype, objectname)
        return dropstatement
{'fraction_non_alphanumeric': 0.09045511292702305, 'fraction_numerical': 0.002950856883441153, 'mean_word_length': 1.522616661895219, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2303302', 'n_tokens_mistral': 4574, 'n_tokens_neox': 4383, 'n_words': 1227}