<commit_before>from __future__ import division
from __future__ import print_function

from future import standard_library

standard_library.install_aliases()
from builtins import object
from builtins import range
from builtins import input
from builtins import zip
from builtins import next
from builtins import str
import sys
import os
import getpass
import zipfile
import gzip
import tarfile
import csv
import re
import requests
from math import ceil
from tqdm import tqdm
from retriever.lib.tools import open_fr, open_fw, open_csvw, walk_relative_path
from setuptools import archive_util
from retriever.lib.defaults import DATA_SEARCH_PATHS, DATA_WRITE_PATH, ENCODING
from retriever.lib.cleanup import no_cleanup
from retriever.lib.warning import Warning
from urllib.request import urlretrieve
from requests.exceptions import InvalidSchema
from imp import reload

encoding = ENCODING.lower()
# sys removes the setdefaultencoding method at startup; reload to get it back
reload(sys)
if hasattr(sys, 'setdefaultencoding'):
    sys.setdefaultencoding(encoding)


class Engine(object):
    """A generic database system. Specific database platforms will inherit
    from this class."""

    _connection = None
    _cursor = None
    datatypes = []
    db = None
    debug = False
    instructions = "Enter your database connection information:"
    name = ""
    pkformat = "%s PRIMARY KEY %s "
    placeholder = None
    required_opts = []
    script = None
    table = None
    use_cache = True
    warnings = []

    def connect(self, force_reconnect=False):
        if force_reconnect:
            self.disconnect()

        if self._connection is None:
            self._connection = self.get_connection()

        return self._connection

    connection = property(connect)

    def disconnect(self):
        if self._connection:
            self.connection.close()
            self._connection = None
            self._cursor = None

    def get_connection(self):
        """This method should be overridden by specific implementations
        of Engine."""
        raise NotImplementedError

    def add_to_table(self, data_source):
        """Adds data to a table from one or more lines specified
        in engine.table.source."""
        if self.table.columns[-1][1][0][:3] == "ct-":
            # cross-tab data
            real_line_length = self.get_ct_line_length(
                gen_from_source(data_source))

            real_lines = self.get_ct_data(
                gen_from_source(data_source))
        else:
            real_lines = gen_from_source(data_source)
            len_source = gen_from_source(data_source)
            real_line_length = sum(1 for _ in len_source)

        total = self.table.record_id + real_line_length
        count_iter = 1
        insert_limit = self.insert_limit
        types = self.table.get_column_datatypes()
        multiple_values = []
        progress_bar = tqdm(
            desc='Installing {}'.format(self.table_name()),
            total=total,
            unit='rows')

        line_values = None
        for line in real_lines:
            if line:
                # Only process non empty lines
                self.table.record_id += 1
                line_values = self.table.values_from_line(line)
                # Build insert statement with the correct number of values
                try:
                    clean_values = [
                        self.format_insert_value(
                            self.table.cleanup.function(
                                line_values[n],
                                self.table.cleanup.args),
                            types[n])
                        for n in range(len(line_values))
                        ]
                except Exception as e:
                    self.warning(
                        'Exception in line {}: {}'
                            .format(self.table.record_id, e))
                    continue

            if line or count_iter == real_line_length:
                if count_iter % insert_limit == 0 or count_iter == real_line_length:
                    # Add values to the list multiple_values
                    # if multiple_values list is full
                    # or we reached the last value in real_line_length
                    # execute the values in multiple_values
                    multiple_values.append(clean_values)
                    try:
                        insert_stmt = self.insert_statement(multiple_values)
                    except Exception as _:
                        if self.debug:
                            print(types)
                        if self.debug:
                            print(line_values)
                        if self.debug:
                            print(clean_values)
                        raise
                    try:
                        self.executemany(insert_stmt,
                                         multiple_values,
                                         commit=False)
                    except:
                        print(insert_stmt)
                        raise
                    multiple_values = []
                else:
                    multiple_values.append(clean_values)
            progress_bar.update()
            count_iter += 1
        progress_bar.close()
        self.connection.commit()

    def get_ct_line_length(self, lines):
        """Returns the number of real lines for cross-tab data"""
        real_line_length = 0
        for values in lines:
            initial_cols = len(self.table.columns) - \
                           (3 if hasattr(self.table, "ct_names") else 2)
            # add one if auto increment is not
            # set to get the right initial columns
            if not self.table.columns[0][1][0] == "pk-auto":
                initial_cols += 1
            rest = values[initial_cols:]
            real_line_length += len(rest)

        return real_line_length

    def get_ct_data(self, lines):
        """Create cross tab data."""
        for values in lines:
            initial_cols = len(self.table.columns) - (3 if hasattr(self.table, "ct_names") else 2)
            # add one if auto increment is not set to get the right initial columns
            if not self.table.columns[0][1][0] == "pk-auto":
                initial_cols += 1
            begin = values[:initial_cols]
            rest = values[initial_cols:]
            n = 0
            for item in rest:
                if hasattr(self.table, "ct_names"):
                    name = [self.table.ct_names[n]]
                    n += 1
                else:
                    name = []
                yield (begin + name + [item])

    def auto_create_table(self, table, url=None, filename=None, pk=None):
        """Create table automatically by analyzing a data source and
        predicting column names, data types, delimiter, etc."""
        if url and not filename:
            filename = filename_from_url(url)
        self.table = table

        if url and not self.find_file(filename):
            # If the file doesn't exist, download it
            self.download_file(url, filename)
        file_path = self.find_file(filename)

        if not self.table.delimiter:
            self.set_table_delimiter(file_path)

        if self.table.header_rows > 0 and not self.table.columns:
            source = (skip_rows,
                      (self.table.header_rows - 1, self.load_data(file_path)))

            lines = gen_from_source(source)
            header = next(lines)
            lines.close()

            source = (skip_rows,
                      (self.table.header_rows, self.load_data(file_path)))

            lines = gen_from_source(source)

            columns, column_values = self.table.auto_get_columns(header)

            self.auto_get_datatypes(pk, lines, columns)

        if self.table.columns[-1][1][0][:3] == "ct-" \
                and hasattr(self.table, "ct_names") \
                and self.table.ct_column not in [c[0] for c in self.table.columns]:
            self.table.columns = self.table.columns[:-1] + \
                                 [(self.table.ct_column, ("char", 50))] + \
                                 [self.table.columns[-1]]

        self.create_table()

    def auto_get_datatypes(self, pk, source, columns):
        """Determine data types for each column.

        For string columns adds an additional 100 characters to the maximum
        observed value to provide extra space for cases where special characters
        are counted differently by different engines.

        """
        # Get all values for each column
        lines_to_scan = source
        # set default column data types as int
        column_types = [('int',)] * len(columns)
        max_lengths = [0] * len(columns)

        # Check the values for each column to determine data type
        for values in lines_to_scan:
            if values:
                for i in range(len(columns)):
                    try:
                        val = u"{}".format(values[i])

                        if self.table.cleanup.function != no_cleanup:
                            val = self.table.cleanup.function(
                                val, self.table.cleanup.args)

                        if val and val.strip():
                            if len(str(val)) + 100 > max_lengths[i]:
                                max_lengths[i] = len(str(val)) + 100

                            if column_types[i][0] in ('int', 'bigint'):
                                try:
                                    val = int(val)
                                    if column_types[i][0] == 'int' and \
                                            hasattr(self, 'max_int') and \
                                                    val > self.max_int:
                                        column_types[i] = ['bigint', ]
                                except Exception as _:
                                    column_types[i] = ['double', ]
                            if column_types[i][0] == 'double':
                                try:
                                    val = float(val)
                                    if "e" in str(val) or ("." in str(val) and len(str(val).split(".")[1]) > 10):
                                        column_types[i] = ["decimal", "50,30"]
                                except Exception as _:
                                    column_types[i] = ['char', max_lengths[i]]
                            if column_types[i][0] == 'char':
                                if len(str(val)) + 100 > column_types[i][1]:
                                    column_types[i][1] = max_lengths[i]
                    except IndexError:
                        pass

        for i in range(len(columns)):
            column = columns[i]
            column[1] = column_types[i]
            if pk == column[0]:
                column[1][0] = "pk-" + column[1][0]
        if pk is None and columns[0][1][0] == 'pk-auto':
            self.table.columns = [("record_id", ("pk-auto",))]
            self.table.contains_pk = True
        else:
            self.table.columns = []

        for column in columns:
            self.table.columns.append((column[0], tuple(column[1])))

    def auto_get_delimiter(self, header):
        """Determine the delimiter.

        Find out which of a set of common delimiters occurs most in the header
        line and use this as the delimiter.

        """
        self.table.delimiter = "\t"
        for other_delimiter in [",", ";"]:
            if header.count(other_delimiter) > header.count(self.table.delimiter):
                self.table.delimiter = other_delimiter

    def convert_data_type(self, datatype):
        """Convert Retriever generic data types to database platform specific
        data types.
        """
        # get the type from the dataset variables
        key = datatype[0]
        this_pk = False
        if key[0:3] == "pk-":
            key = key[3:]
            this_pk = True
        elif key[0:3] == "ct-":
            key = key[3:]

        # format the dataset type to match engine specific type
        this_type = ""
        if key in list(self.datatypes.keys()):
            this_type = self.datatypes[key]
            if isinstance(this_type, tuple):
                if datatype[0] == 'pk-auto':
                    pass
                elif len(datatype) > 1:
                    this_type = this_type[1] + "(" + str(datatype[1]) + ")"
                else:
                    this_type = this_type[0]
            else:
                if len(datatype) > 1:
                    this_type += "(" + str(datatype[1]) + ")"

        # set the PRIMARY KEY
        if this_pk:
            if isinstance(this_type, tuple):
                this_type = self.pkformat % this_type
            else:
                this_type = self.pkformat % (this_type, "")
        return this_type

    def create_db(self):
        """Create a new database based on settings supplied in Database object
        engine.db."""
        db_name = self.database_name()
        if db_name:
            print("Creating database " + db_name + "...\n")
            # Create the database
            create_stmt = self.create_db_statement()
            if self.debug:
                print(create_stmt)
            try:
                self.execute(create_stmt)
            except Exception as e:
                try:
                    self.connection.rollback()
                except Exception as _:
                    pass
                print("Couldn't create database (%s). Trying to continue anyway." % e)

    def create_db_statement(self):
        """Return SQL statement to create a database."""
        create_stmt = "CREATE DATABASE " + self.database_name()
        return create_stmt

    def create_raw_data_dir(self, path=None):
        """Check to see if the archive directory exists and creates it if
        necessary."""
        if not path:
            path = self.format_data_dir()
        if not os.path.exists(path):
            os.makedirs(path)

    def create_table(self):
        """Create new database table based on settings supplied in Table
        object engine.table."""

        # Try to drop the table if it exists; this may cause an exception if it
        # doesn't exist, so ignore exceptions
        try:
            self.execute(self.drop_statement("TABLE", self.table_name()))
        except Exception as _:
            pass

        create_stmt = self.create_table_statement()
        if self.debug:
            print(create_stmt)
        try:
            self.execute(create_stmt)
            if self.table.name not in self.script.tables:
                self.script.tables[self.table.name] = self.table
        except Exception as e:
            try:
                self.connection.rollback()
            except Exception as _:
                pass
            print("Couldn't create table (%s). Trying to continue anyway." % e)

    def create_table_statement(self):
        """Return SQL statement to create a table."""
        create_stmt = "CREATE TABLE " + self.table_name() + " ("
        columns = self.table.get_insert_columns(join=False, create=True)
        types = []
        for column in self.table.columns:
            for column_name in columns:
                if column[0] == column_name:
                    types.append(self.convert_data_type(column[1]))
        if self.debug:
            print(columns)

        column_strings = []
        for c, t in zip(columns, types):
            column_strings.append(c + ' ' + t)

        create_stmt += ', '.join(column_strings)
        create_stmt += " );"

        return create_stmt

    def database_name(self, name=None):
        """Return name of the database."""
        if not name:
            try:
                name = self.script.name
            except AttributeError:
                name = "{db}"
        try:
            db_name = self.opts["database_name"].format(db=name)
        except KeyError:
            db_name = name
        return db_name.replace('-', '_')

    def download_file(self, url, filename):
        """Download file to the raw data directory."""
        if not self.find_file(filename) or not self.use_cache:
            path = self.format_filename(filename)
            self.create_raw_data_dir()
            progbar = tqdm(unit='B',
                           unit_scale=True,
                           unit_divisor=1024,
                           miniters=1,
                           desc='Downloading {}'.format(filename))

            try:
                requests.get(url, allow_redirects=True,
                             stream=True,
                             headers={'user-agent': 'Weecology/Data-Retriever \
                                            Package Manager: http://www.data-retriever.org/'},
                             hooks={'response': reporthook(progbar, path)})

            except InvalidSchema:
                urlretrieve(url, path, reporthook=reporthook(progbar))

            self.use_cache = True
            progbar.close()

    def download_files_from_archive(self, url,
                                    file_names=None, archive_type="zip",
                                    keep_in_dir=False, archive_name=None):
        """Download files from an archive into the raw data directory."""

        if not archive_name:
            archive_name = filename_from_url(url)
        else:
            archive_name = self.format_filename(archive_name)

        archive_full_path = self.format_filename(archive_name)
        archive_dir = self.format_data_dir()
        if keep_in_dir:
            archive_base = os.path.splitext(os.path.basename(archive_name))[0]
            archive_dir = os.path.join(DATA_WRITE_PATH, archive_base)
            archive_dir = archive_dir.format(dataset=self.script.name)
            if not os.path.exists(archive_dir):
                os.makedirs(archive_dir)

        if not file_names:
            self.download_file(url, archive_name)
            if archive_type == 'tar' or archive_type == 'tar.gz':
                file_names = self.extract_tar(archive_full_path, archive_dir, archive_type)
            elif archive_type == 'zip':
                file_names = self.extract_zip(archive_full_path, archive_dir)
            elif archive_type == 'gz':
                file_names = self.extract_gz(archive_full_path, archive_dir)
            return file_names

        if file_names:
            archive_downloaded = False
            for file_name in file_names:
                archive_full_path = self.format_filename(archive_name)
                if not self.find_file(os.path.join(archive_dir, file_name)):
                    # if no local copy, download the data
                    self.create_raw_data_dir()
                    if not archive_downloaded:
                        self.download_file(url, archive_name)
                        archive_downloaded = True
                    if archive_type == 'zip':
                        self.extract_zip(archive_full_path, archive_dir, file_name)
                    elif archive_type == 'gz':
                        self.extract_gz(archive_full_path, archive_dir, file_name)
                    elif archive_type == 'tar' or archive_type == 'tar.gz':
                        self.extract_tar(archive_full_path, archive_dir, archive_type, file_name)
            return file_names

    def drop_statement(self, object_type, object_name):
        """Return drop table or database SQL statement."""
        if self:
            drop_statement = "DROP %s IF EXISTS %s" % (object_type, object_name)
        return drop_statement

    def execute(self, statement, commit=True):
        """Execute given statement."""
        self.cursor.execute(statement)
        if commit:
            self.connection.commit()

    def executemany(self, statement, values, commit=True):
        """Execute given statement with multiple values."""
        self.cursor.executemany(statement, values)<commit_msg>Refactor extract methods from download_from_archive.<commit_after>from __future__ import division
from __future__ import print_function

from future import standard_library

standard_library.install_aliases()
from builtins import object
from builtins import range
from builtins import input
from builtins import zip
from builtins import next
from builtins import str
import sys
import os
import getpass
import zipfile
import gzip
import tarfile
import csv
import re
import requests
from math import ceil
from tqdm import tqdm
from retriever.lib.tools import open_fr, open_fw, open_csvw, walk_relative_path
from setuptools import archive_util
from retriever.lib.defaults import DATA_SEARCH_PATHS, DATA_WRITE_PATH, ENCODING
from retriever.lib.cleanup import no_cleanup
from retriever.lib.warning import Warning
from urllib.request import urlretrieve
from requests.exceptions import InvalidSchema
from imp import reload

encoding = ENCODING.lower()
# sys removes the setdefaultencoding method at startup; reload to get it back
reload(sys)
if hasattr(sys, 'setdefaultencoding'):
    sys.setdefaultencoding(encoding)


class Engine(object):
    """A generic database system. Specific database platforms will inherit
    from this class."""

    _connection = None
    _cursor = None
    datatypes = []
    db = None
    debug = False
    instructions = "Enter your database connection information:"
    name = ""
    pkformat = "%s PRIMARY KEY %s "
    placeholder = None
    required_opts = []
    script = None
    table = None
    use_cache = True
    warnings = []

    def connect(self, force_reconnect=False):
        if force_reconnect:
            self.disconnect()

        if self._connection is None:
            self._connection = self.get_connection()

        return self._connection

    connection = property(connect)

    def disconnect(self):
        if self._connection:
            self.connection.close()
            self._connection = None
            self._cursor = None

    def get_connection(self):
        """This method should be overridden by specific implementations
        of Engine."""
        raise NotImplementedError

    def add_to_table(self, data_source):
        """Adds data to a table from one or more lines specified
        in engine.table.source."""
        if self.table.columns[-1][1][0][:3] == "ct-":
            # cross-tab data
            real_line_length = self.get_ct_line_length(
                gen_from_source(data_source))

            real_lines = self.get_ct_data(
                gen_from_source(data_source))
        else:
            real_lines = gen_from_source(data_source)
            len_source = gen_from_source(data_source)
            real_line_length = sum(1 for _ in len_source)

        total = self.table.record_id + real_line_length
        count_iter = 1
        insert_limit = self.insert_limit
        types = self.table.get_column_datatypes()
        multiple_values = []
        progress_bar = tqdm(
            desc='Installing {}'.format(self.table_name()),
            total=total,
            unit='rows')

        line_values = None
        for line in real_lines:
            if line:
                # Only process non empty lines
                self.table.record_id += 1
                line_values = self.table.values_from_line(line)
                # Build insert statement with the correct number of values
                try:
                    clean_values = [
                        self.format_insert_value(
                            self.table.cleanup.function(
                                line_values[n],
                                self.table.cleanup.args),
                            types[n])
                        for n in range(len(line_values))
                        ]
                except Exception as e:
                    self.warning(
                        'Exception in line {}: {}'
                            .format(self.table.record_id, e))
                    continue

            if line or count_iter == real_line_length:
                if count_iter % insert_limit == 0 or count_iter == real_line_length:
                    # Add values to the list multiple_values
                    # if multiple_values list is full
                    # or we reached the last value in real_line_length
                    # execute the values in multiple_values
                    multiple_values.append(clean_values)
                    try:
                        insert_stmt = self.insert_statement(multiple_values)
                    except Exception as _:
                        if self.debug:
                            print(types)
                        if self.debug:
                            print(line_values)
                        if self.debug:
                            print(clean_values)
                        raise
                    try:
                        self.executemany(insert_stmt,
                                         multiple_values,
                                         commit=False)
                    except:
                        print(insert_stmt)
                        raise
                    multiple_values = []
                else:
                    multiple_values.append(clean_values)
            progress_bar.update()
            count_iter += 1
        progress_bar.close()
        self.connection.commit()

    def get_ct_line_length(self, lines):
        """Returns the number of real lines for cross-tab data"""
        real_line_length = 0
        for values in lines:
            initial_cols = len(self.table.columns) - \
                           (3 if hasattr(self.table, "ct_names") else 2)
            # add one if auto increment is not
            # set to get the right initial columns
            if not self.table.columns[0][1][0] == "pk-auto":
                initial_cols += 1
            rest = values[initial_cols:]
            real_line_length += len(rest)

        return real_line_length

    def get_ct_data(self, lines):
        """Create cross tab data."""
        for values in lines:
            initial_cols = len(self.table.columns) - (3 if hasattr(self.table, "ct_names") else 2)
            # add one if auto increment is not set to get the right initial columns
            if not self.table.columns[0][1][0] == "pk-auto":
                initial_cols += 1
            begin = values[:initial_cols]
            rest = values[initial_cols:]
            n = 0
            for item in rest:
                if hasattr(self.table, "ct_names"):
                    name = [self.table.ct_names[n]]
                    n += 1
                else:
                    name = []
                yield (begin + name + [item])

    def auto_create_table(self, table, url=None, filename=None, pk=None):
        """Create table automatically by analyzing a data source and
        predicting column names, data types, delimiter, etc."""
        if url and not filename:
            filename = filename_from_url(url)
        self.table = table

        if url and not self.find_file(filename):
            # If the file doesn't exist, download it
            self.download_file(url, filename)
        file_path = self.find_file(filename)

        if not self.table.delimiter:
            self.set_table_delimiter(file_path)

        if self.table.header_rows > 0 and not self.table.columns:
            source = (skip_rows,
                      (self.table.header_rows - 1, self.load_data(file_path)))

            lines = gen_from_source(source)
            header = next(lines)
            lines.close()

            source = (skip_rows,
                      (self.table.header_rows, self.load_data(file_path)))

            lines = gen_from_source(source)

            columns, column_values = self.table.auto_get_columns(header)

            self.auto_get_datatypes(pk, lines, columns)

        if self.table.columns[-1][1][0][:3] == "ct-" \
                and hasattr(self.table, "ct_names") \
                and self.table.ct_column not in [c[0] for c in self.table.columns]:
            self.table.columns = self.table.columns[:-1] + \
                                 [(self.table.ct_column, ("char", 50))] + \
                                 [self.table.columns[-1]]

        self.create_table()

    def auto_get_datatypes(self, pk, source, columns):
        """Determine data types for each column.

        For string columns adds an additional 100 characters to the maximum
        observed value to provide extra space for cases where special characters
        are counted differently by different engines.

        """
        # Get all values for each column
        lines_to_scan = source
        # set default column data types as int
        column_types = [('int',)] * len(columns)
        max_lengths = [0] * len(columns)

        # Check the values for each column to determine data type
        for values in lines_to_scan:
            if values:
                for i in range(len(columns)):
                    try:
                        val = u"{}".format(values[i])

                        if self.table.cleanup.function != no_cleanup:
                            val = self.table.cleanup.function(
                                val, self.table.cleanup.args)

                        if val and val.strip():
                            if len(str(val)) + 100 > max_lengths[i]:
                                max_lengths[i] = len(str(val)) + 100

                            if column_types[i][0] in ('int', 'bigint'):
                                try:
                                    val = int(val)
                                    if column_types[i][0] == 'int' and \
                                            hasattr(self, 'max_int') and \
                                                    val > self.max_int:
                                        column_types[i] = ['bigint', ]
                                except Exception as _:
                                    column_types[i] = ['double', ]
                            if column_types[i][0] == 'double':
                                try:
                                    val = float(val)
                                    if "e" in str(val) or ("." in str(val) and len(str(val).split(".")[1]) > 10):
                                        column_types[i] = ["decimal", "50,30"]
                                except Exception as _:
                                    column_types[i] = ['char', max_lengths[i]]
                            if column_types[i][0] == 'char':
                                if len(str(val)) + 100 > column_types[i][1]:
                                    column_types[i][1] = max_lengths[i]
                    except IndexError:
                        pass

        for i in range(len(columns)):
            column = columns[i]
            column[1] = column_types[i]
            if pk == column[0]:
                column[1][0] = "pk-" + column[1][0]
        if pk is None and columns[0][1][0] == 'pk-auto':
            self.table.columns = [("record_id", ("pk-auto",))]
            self.table.contains_pk = True
        else:
            self.table.columns = []

        for column in columns:
            self.table.columns.append((column[0], tuple(column[1])))

    def auto_get_delimiter(self, header):
        """Determine the delimiter.

        Find out which of a set of common delimiters occurs most in the header
        line and use this as the delimiter.

        """
        self.table.delimiter = "\t"
        for other_delimiter in [",", ";"]:
            if header.count(other_delimiter) > header.count(self.table.delimiter):
                self.table.delimiter = other_delimiter

    def convert_data_type(self, datatype):
        """Convert Retriever generic data types to database platform specific
        data types.
        """
        # get the type from the dataset variables
        key = datatype[0]
        this_pk = False
        if key[0:3] == "pk-":
            key = key[3:]
            this_pk = True
        elif key[0:3] == "ct-":
            key = key[3:]

        # format the dataset type to match engine specific type
        this_type = ""
        if key in list(self.datatypes.keys()):
            this_type = self.datatypes[key]
            if isinstance(this_type, tuple):
                if datatype[0] == 'pk-auto':
                    pass
                elif len(datatype) > 1:
                    this_type = this_type[1] + "(" + str(datatype[1]) + ")"
                else:
                    this_type = this_type[0]
            else:
                if len(datatype) > 1:
                    this_type += "(" + str(datatype[1]) + ")"

        # set the PRIMARY KEY
        if this_pk:
            if isinstance(this_type, tuple):
                this_type = self.pkformat % this_type
            else:
                this_type = self.pkformat % (this_type, "")
        return this_type

    def create_db(self):
        """Create a new database based on settings supplied in Database object
        engine.db."""
        db_name = self.database_name()
        if db_name:
            print("Creating database " + db_name + "...\n")
            # Create the database
            create_stmt = self.create_db_statement()
            if self.debug:
                print(create_stmt)
            try:
                self.execute(create_stmt)
            except Exception as e:
                try:
                    self.connection.rollback()
                except Exception as _:
                    pass
                print("Couldn't create database (%s). Trying to continue anyway." % e)

    def create_db_statement(self):
        """Return SQL statement to create a database."""
        create_stmt = "CREATE DATABASE " + self.database_name()
        return create_stmt

    def create_raw_data_dir(self, path=None):
        """Check to see if the archive directory exists and creates it if
        necessary."""
        if not path:
            path = self.format_data_dir()
        if not os.path.exists(path):
            os.makedirs(path)

    def create_table(self):
        """Create new database table based on settings supplied in Table
        object engine.table."""

        # Try to drop the table if it exists; this may cause an exception if it
        # doesn't exist, so ignore exceptions
        try:
            self.execute(self.drop_statement("TABLE", self.table_name()))
        except Exception as _:
            pass

        create_stmt = self.create_table_statement()
        if self.debug:
            print(create_stmt)
        try:
            self.execute(create_stmt)
            if self.table.name not in self.script.tables:
                self.script.tables[self.table.name] = self.table
        except Exception as e:
            try:
                self.connection.rollback()
            except Exception as _:
                pass
            print("Couldn't create table (%s). Trying to continue anyway." % e)

    def create_table_statement(self):
        """Return SQL statement to create a table."""
        create_stmt = "CREATE TABLE " + self.table_name() + " ("
        columns = self.table.get_insert_columns(join=False, create=True)
        types = []
        for column in self.table.columns:
            for column_name in columns:
                if column[0] == column_name:
                    types.append(self.convert_data_type(column[1]))
        if self.debug:
            print(columns)

        column_strings = []
        for c, t in zip(columns, types):
            column_strings.append(c + ' ' + t)

        create_stmt += ', '.join(column_strings)
        create_stmt += " );"

        return create_stmt

    def database_name(self, name=None):
        """Return name of the database."""
        if not name:
            try:
                name = self.script.name
            except AttributeError:
                name = "{db}"
        try:
            db_name = self.opts["database_name"].format(db=name)
        except KeyError:
            db_name = name
        return db_name.replace('-', '_')

    def download_file(self, url, filename):
        """Download file to the raw data directory."""
        if not self.find_file(filename) or not self.use_cache:
            path = self.format_filename(filename)
            self.create_raw_data_dir()
            progbar = tqdm(unit='B',
                           unit_scale=True,
                           unit_divisor=1024,
                           miniters=1,
                           desc='Downloading {}'.format(filename))

            try:
                requests.get(url, allow_redirects=True,
                             stream=True,
                             headers={'user-agent': 'Weecology/Data-Retriever \
                                            Package Manager: http://www.data-retriever.org/'},
                             hooks={'response': reporthook(progbar, path)})

            except InvalidSchema:
                urlretrieve(url, path, reporthook=reporthook(progbar))

            self.use_cache = True
            progbar.close()

    def download_files_from_archive(self, url,
                                    file_names=None, archive_type="zip",
                                    keep_in_dir=False, archive_name=None):
        """Download files from an archive into the raw data directory."""

        if not archive_name:
            archive_name = filename_from_url(url)
        else:
            archive_name = self.format_filename(archive_name)

        archive_full_path = self.format_filename(archive_name)
        archive_dir = self.format_data_dir()
        if keep_in_dir:
            archive_base = os.path.splitext(os.path.basename(archive_name))[0]
            archive_dir = os.path.join(DATA_WRITE_PATH, archive_base)
            archive_dir = archive_dir.format(dataset=self.script.name)
            if not os.path.exists(archive_dir):
                os.makedirs(archive_dir)

        if not file_names:
            self.download_file(url, archive_name)
            if archive_type == 'tar' or archive_type == 'tar.gz':
                file_names = self.extract_tar(archive_full_path, archive_dir, archive_type)
            elif archive_type == 'zip':
                file_names = self.extract_zip(archive_full_path, archive_dir)
            elif archive_type == 'gz':
                file_names = self.extract_gz(archive_full_path, archive_dir)
            return file_names

        archive_downloaded = False
        for file_name in file_names:
            archive_full_path = self.format_filename(archive_name)
            if not self.find_file(os.path.join(archive_dir, file_name)):
                # if no local copy, download the data
                self.create_raw_data_dir()
                if not archive_downloaded:
                    self.download_file(url, archive_name)
                    archive_downloaded = True
                if archive_type == 'zip':
                    self.extract_zip(archive_full_path, archive_dir, file_name)
                elif archive_type == 'gz':
                    self.extract_gz(archive_full_path, archive_dir, file_name)
                elif archive_type == 'tar' or archive_type == 'tar.gz':
                    self.extract_tar(archive_full_path, archive_dir, archive_type, file_name)
        return file_names

    def drop_statement(self, object_type, object_name):
        """Return drop table or database SQL statement."""
        if self:
            drop_statement = "DROP %s IF EXISTS %s" % (object_type, object_name)
        return drop_statement

    def execute(self, statement, commit=True):
        """Execute given statement."""
        self.cursor.execute(statement)
        if commit:
            self.connection.commit()

    def executemany(self, statement, values, commit=True):
        """Execute given statement with multiple values."""
        self.cursor.executemany(statement, values)
{'fraction_non_alphanumeric': 0.08717489681237257, 'fraction_numerical': 0.004674523845044507, 'mean_word_length': 1.5587861051024303, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 77, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '736227', 'n_tokens_mistral': 10635, 'n_tokens_neox': 10145, 'n_words': 2739}