<commit_before>import sys
import os
import getpass
import zipfile
import urllib
import csv
import itertools
from decimal import Decimal
from retriever import DATA_SEARCH_PATHS, DATA_WRITE_PATH
from retriever.lib.cleanup import no_cleanup


class Engine():
    """A generic database system. Specific database platforms will inherit 
    from this class."""
    name = ""
    instructions = "Enter your database connection information:"
    db = None
    table = None
    connection = None
    cursor = None
    datatypes = []
    required_opts = []
    pkformat = "%s PRIMARY KEY"
    script = None
    debug = False

    
    def add_to_table(self):
        """This function adds data to a table from one or more lines specified 
        in engine.table.source."""
        self.get_cursor()
        
        if self.table.columns[-1][1][0][:3] == "ct-":        
            # cross-tab data
            
            lines = Engine.gen_from_source(self.table.source)
            real_lines = []
            for line in lines:
                split_line = line.strip('\n\r\t ').split(self.table.delimiter)
                begin = split_line[:len(self.table.columns) - (3 if hasattr(self.table, "ct_names") else 2)]
                rest = split_line[len(self.table.columns) - 2:]
                n = 0
                for item in rest:
                    if hasattr(self.table, "ct_names"):
                        name = [self.table.ct_names[n]]
                        n += 1
                    else:
                        name = []
                    real_lines.append(self.table.delimiter.join(begin + name + [item]))
            real_line_length = len(real_lines)
        else:
            def source_gen():
                return (line for line in Engine.gen_from_source(self.table.source)
                         if line.strip('\n\r\t '))
            real_lines, len_source = source_gen(), source_gen()
            real_line_length = sum(1 for _ in len_source)
            
        total = self.table.record_id + real_line_length
        for line in real_lines:
            line = line.strip()
            if line:
                self.table.record_id += 1            
                linevalues = self.values_from_line(line)
                
                types = self.get_column_datatypes()            
                # Build insert statement with the correct # of values                
                cleanvalues = [self.format_insert_value(self.table.cleanup.function
                                                        (linevalues[n], 
                                                         self.table.cleanup.args),
                                                        types[n]) 
                               for n in range(len(linevalues))]
                try:
                    insert_stmt = self.insert_statement(cleanvalues)
                except:
                    if self.debug: print types
                    if self.debug: print linevalues
                    if self.debug: print cleanvalues
                    raise
                
                try:
                    update_frequency = int(self.update_frequency)
                except:
                    update_frequency = 100
                    
                if (self.table.record_id % update_frequency == 0 
                    or self.table.record_id == 1 
                    or self.table.record_id == total):
                    prompt = "Inserting rows to " + self.tablename() + ": "
                    prompt += str(self.table.record_id) + " / " + str(total)
                    sys.stdout.write(prompt + "\b" * len(prompt))
                
                try:
                    self.execute(insert_stmt, commit=False)
                except:
                    print insert_stmt
                    raise
        
        print
        self.connection.commit()

        
    def auto_create_table(self, table, url=None, filename=None, pk=None):
        """Creates a table automatically by analyzing a data source and 
        predicting column names, data types, delimiter, etc."""
        if url and not filename:
            filename = filename_from_url(url)
        self.table = table
        
        if url and not self.find_file(filename):
            # If the file doesn't exist, download it
            self.download_file(url, filename)
        file_path = self.find_file(filename)

        source = (self.skip_rows,
                  (self.table.column_names_row - 1, 
                   (open, (file_path, "rb"))))
        lines = Engine.gen_from_source(source)

        header = lines.next()
        lines.close()

        if not self.table.delimiter:
            self.auto_get_delimiter(header)
        
        if not self.table.columns:            
            lines = Engine.gen_from_source(source)
            
            if pk is None:
                self.table.columns = [("record_id", ("pk-auto",))]
            else:
                self.table.columns = []
                self.table.contains_pk = True
                
            columns, column_values = self.auto_get_columns(header)
            
            self.auto_get_datatypes(pk, lines, columns, column_values)

        if self.table.columns[-1][1][0][:3] == "ct-" and hasattr(self.table, "ct_names") and not self.table.ct_column in [c[0] for c in self.table.columns]:
            self.table.columns = self.table.columns[:-1] + [(self.table.ct_column, ("char", 20))] + [self.table.columns[-1]]

        
        self.create_table()
<commit_msg>Bug in auto_create_table results in all fields being text, because header row is being treated as the first row of data.<commit_after>import sys
import os
import getpass
import zipfile
import urllib
import csv
import itertools
from decimal import Decimal
from retriever import DATA_SEARCH_PATHS, DATA_WRITE_PATH
from retriever.lib.cleanup import no_cleanup


class Engine():
    """A generic database system. Specific database platforms will inherit 
    from this class."""
    name = ""
    instructions = "Enter your database connection information:"
    db = None
    table = None
    connection = None
    cursor = None
    datatypes = []
    required_opts = []
    pkformat = "%s PRIMARY KEY"
    script = None
    debug = False

    
    def add_to_table(self):
        """This function adds data to a table from one or more lines specified 
        in engine.table.source."""
        self.get_cursor()
        
        if self.table.columns[-1][1][0][:3] == "ct-":        
            # cross-tab data
            
            lines = Engine.gen_from_source(self.table.source)
            real_lines = []
            for line in lines:
                split_line = line.strip('\n\r\t ').split(self.table.delimiter)
                begin = split_line[:len(self.table.columns) - (3 if hasattr(self.table, "ct_names") else 2)]
                rest = split_line[len(self.table.columns) - 2:]
                n = 0
                for item in rest:
                    if hasattr(self.table, "ct_names"):
                        name = [self.table.ct_names[n]]
                        n += 1
                    else:
                        name = []
                    real_lines.append(self.table.delimiter.join(begin + name + [item]))
            real_line_length = len(real_lines)
        else:
            def source_gen():
                return (line for line in Engine.gen_from_source(self.table.source)
                         if line.strip('\n\r\t '))
            real_lines, len_source = source_gen(), source_gen()
            real_line_length = sum(1 for _ in len_source)
            
        total = self.table.record_id + real_line_length
        for line in real_lines:
            line = line.strip()
            if line:
                self.table.record_id += 1            
                linevalues = self.values_from_line(line)
                
                types = self.get_column_datatypes()            
                # Build insert statement with the correct # of values                
                cleanvalues = [self.format_insert_value(self.table.cleanup.function
                                                        (linevalues[n], 
                                                         self.table.cleanup.args),
                                                        types[n]) 
                               for n in range(len(linevalues))]
                try:
                    insert_stmt = self.insert_statement(cleanvalues)
                except:
                    if self.debug: print types
                    if self.debug: print linevalues
                    if self.debug: print cleanvalues
                    raise
                
                try:
                    update_frequency = int(self.update_frequency)
                except:
                    update_frequency = 100
                    
                if (self.table.record_id % update_frequency == 0 
                    or self.table.record_id == 1 
                    or self.table.record_id == total):
                    prompt = "Inserting rows to " + self.tablename() + ": "
                    prompt += str(self.table.record_id) + " / " + str(total)
                    sys.stdout.write(prompt + "\b" * len(prompt))
                
                try:
                    self.execute(insert_stmt, commit=False)
                except:
                    print insert_stmt
                    raise
        
        print
        self.connection.commit()

        
    def auto_create_table(self, table, url=None, filename=None, pk=None):
        """Creates a table automatically by analyzing a data source and 
        predicting column names, data types, delimiter, etc."""
        if url and not filename:
            filename = filename_from_url(url)
        self.table = table
        
        if url and not self.find_file(filename):
            # If the file doesn't exist, download it
            self.download_file(url, filename)
        file_path = self.find_file(filename)

        source = (self.skip_rows,
                  (self.table.column_names_row - 1, 
                   (open, (file_path, "rb"))))
        lines = Engine.gen_from_source(source)

        header = lines.next()
        lines.close()

        source = (self.skip_rows,
                  (self.table.header_rows, 
                   (open, (file_path, "rb"))))
                   
        if not self.table.delimiter:
            self.auto_get_delimiter(header)
        
        if not self.table.columns:            
            lines = Engine.gen_from_source(source)
            
            if pk is None:
                self.table.columns = [("record_id", ("pk-auto",))]
            else:
                self.table.columns = []
                self.table.contains_pk = True
                
            columns, column_values = self.auto_get_columns(header)
            
            self.auto_get_datatypes(pk, lines, columns, column_values)

        if self.table.columns[-1][1][0][:3] == "ct-" and hasattr(self.table, "ct_names") and not self.table.ct_column in [c[0] for c in self.table.columns]:
            self.table.columns = self.table.columns[:-1] + [(self.table.ct_column, ("char", 20))] + [self.table.columns[-1]]

        
        self.create_table()

{'fraction_non_alphanumeric': 0.09082907938743179, 'fraction_numerical': 0.004576659038901602, 'mean_word_length': 1.3712437395659431, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '173954', 'n_tokens_mistral': 3031, 'n_tokens_neox': 2871, 'n_words': 718}