<commit_before>from __future__ import print_function
from __future__ import division
from future import standard_library
standard_library.install_aliases()
from builtins import object
from builtins import range
from builtins import input
from builtins import zip
from builtins import next
from builtins import str
import sys
import os
import getpass
import zipfile
import gzip
import tarfile
import csv
import io
if sys.version_info[0] >= 3:
    from urllib.request import urlretrieve
else:
    from urllib import urlretrieve
from retriever import DATA_SEARCH_PATHS, DATA_WRITE_PATH
from retriever.lib.cleanup import no_cleanup
from retriever.lib.warning import Warning


class Engine(object):
    """A generic database system. Specific database platforms will inherit
    from this class."""
    name = ""
    instructions = "Enter your database connection information:"
    db = None
    table = None
    _connection = None
    _cursor = None
    datatypes = []
    required_opts = []
    pkformat = "%s PRIMARY KEY %s "
    script = None
    debug = False
    warnings = []

    def connect(self, force_reconnect=False):
        if force_reconnect:
            self.disconnect()

        if self._connection is None:
            self._connection = self.get_connection()

        return self._connection

    connection = property(connect)

    def disconnect(self):
        if self._connection:
            self.connection.close()
            self._connection = None
            self._cursor = None

    def get_connection(self):
        '''This method should be overloaded by specific implementations
        of Engine.'''
        pass

    def add_to_table(self, data_source):
        """This function adds data to a table from one or more lines specified
        in engine.table.source."""
        if self.table.columns[-1][1][0][:3] == "ct-":
            # cross-tab data

            lines = gen_from_source(data_source)
            real_lines = []
            for line in lines:
                split_line = self.table.split_on_delimiter(line)
                initial_cols = len(self.table.columns) - (3 if hasattr(self.table, "ct_names") else 2)
                # add one if auto increment is not set to get the right initial columns
                if not self.table.columns[0][1][0] == "pk-auto":
                    initial_cols += 1
                begin = split_line[:initial_cols]
                rest = split_line[initial_cols:]
                n = 0
                for item in rest:
                    if hasattr(self.table, "ct_names"):
                        name = [self.table.ct_names[n]]
                        n += 1
                    else:
                        name = []
                    real_lines.append(
                        self.table.combine_on_delimiter(begin + name + [item]))
            real_line_length = len(real_lines)
        else:
            # this function returns a generator that iterates over the lines in
            # the source data
            def source_gen():
                return (line for line in gen_from_source(data_source)
                        if line.strip('\n\r\t '))

            # use one generator to compute the length of the input
            real_lines, len_source = source_gen(), source_gen()
            real_line_length = sum(1 for _ in len_source)

        total = self.table.record_id + real_line_length
        pos = 0
        for line in real_lines:
            if not self.table.fixed_width:
                line = line.strip()
            if line:
                self.table.record_id += 1

                # Check for single row distributed over multiple lines
                val_list = self.table.split_on_delimiter(line)
                while len(val_list) < len(self.table.get_column_datatypes()):
                    line = line.rstrip('\n')
                    if type(real_lines) != 'list':
                        line += next(real_lines)
                    else:
                        line += real_lines[pos+1]
                        real_lines.pop(pos+1)
                    val_list = (self.table.split_on_delimiter(line))
                pos += 1

                linevalues = self.table.values_from_line(line)

                types = self.table.get_column_datatypes()
                # Build insert statement with the correct # of values
                try:
                    cleanvalues = [self.format_insert_value(self.table.cleanup.function
                                                            (linevalues[n],
                                                             self.table.cleanup.args),
                                                            types[n])
                                   for n in range(len(linevalues))]
                except Exception as e:
                    self.warning('Exception in line %s: %s' % (self.table.record_id, e))
                    continue
                try:
                    insert_stmt = self.insert_statement(cleanvalues)
                except:
                    if self.debug:
                        print(types)
                    if self.debug:
                        print(linevalues)
                    if self.debug:
                        print(cleanvalues)<commit_msg>Use extract_values when checking for mid-row line endings<commit_after>from __future__ import print_function
from __future__ import division
from future import standard_library
standard_library.install_aliases()
from builtins import object
from builtins import range
from builtins import input
from builtins import zip
from builtins import next
from builtins import str
import sys
import os
import getpass
import zipfile
import gzip
import tarfile
import csv
import io
if sys.version_info[0] >= 3:
    from urllib.request import urlretrieve
else:
    from urllib import urlretrieve
from retriever import DATA_SEARCH_PATHS, DATA_WRITE_PATH
from retriever.lib.cleanup import no_cleanup
from retriever.lib.warning import Warning


class Engine(object):
    """A generic database system. Specific database platforms will inherit
    from this class."""
    name = ""
    instructions = "Enter your database connection information:"
    db = None
    table = None
    _connection = None
    _cursor = None
    datatypes = []
    required_opts = []
    pkformat = "%s PRIMARY KEY %s "
    script = None
    debug = False
    warnings = []

    def connect(self, force_reconnect=False):
        if force_reconnect:
            self.disconnect()

        if self._connection is None:
            self._connection = self.get_connection()

        return self._connection

    connection = property(connect)

    def disconnect(self):
        if self._connection:
            self.connection.close()
            self._connection = None
            self._cursor = None

    def get_connection(self):
        '''This method should be overloaded by specific implementations
        of Engine.'''
        pass

    def add_to_table(self, data_source):
        """This function adds data to a table from one or more lines specified
        in engine.table.source."""
        if self.table.columns[-1][1][0][:3] == "ct-":
            # cross-tab data

            lines = gen_from_source(data_source)
            real_lines = []
            for line in lines:
                split_line = self.table.split_on_delimiter(line)
                initial_cols = len(self.table.columns) - (3 if hasattr(self.table, "ct_names") else 2)
                # add one if auto increment is not set to get the right initial columns
                if not self.table.columns[0][1][0] == "pk-auto":
                    initial_cols += 1
                begin = split_line[:initial_cols]
                rest = split_line[initial_cols:]
                n = 0
                for item in rest:
                    if hasattr(self.table, "ct_names"):
                        name = [self.table.ct_names[n]]
                        n += 1
                    else:
                        name = []
                    real_lines.append(
                        self.table.combine_on_delimiter(begin + name + [item]))
            real_line_length = len(real_lines)
        else:
            # this function returns a generator that iterates over the lines in
            # the source data
            def source_gen():
                return (line for line in gen_from_source(data_source)
                        if line.strip('\n\r\t '))

            # use one generator to compute the length of the input
            real_lines, len_source = source_gen(), source_gen()
            real_line_length = sum(1 for _ in len_source)

        total = self.table.record_id + real_line_length
        pos = 0
        for line in real_lines:
            if not self.table.fixed_width:
                line = line.strip()
            if line:
                self.table.record_id += 1

                # Check for single row distributed over multiple lines
                val_list = self.table.extract_values(line)
                while len(val_list) < len(self.table.get_column_datatypes()):
                    line = line.rstrip('\n')
                    if type(real_lines) != 'list':
                        line += next(real_lines)
                    else:
                        line += real_lines[pos+1]
                        real_lines.pop(pos+1)
                    val_list = (self.table.extract_values(line))
                pos += 1

                linevalues = self.table.values_from_line(line)

                types = self.table.get_column_datatypes()
                # Build insert statement with the correct # of values
                try:
                    cleanvalues = [self.format_insert_value(self.table.cleanup.function
                                                            (linevalues[n],
                                                             self.table.cleanup.args),
                                                            types[n])
                                   for n in range(len(linevalues))]
                except Exception as e:
                    self.warning('Exception in line %s: %s' % (self.table.record_id, e))
                    continue
                try:
                    insert_stmt = self.insert_statement(cleanvalues)
                except:
                    if self.debug:
                        print(types)
                    if self.debug:
                        print(linevalues)
                    if self.debug:
                        print(cleanvalues)
{'fraction_non_alphanumeric': 0.07387438516836928, 'fraction_numerical': 0.0037835792659856224, 'mean_word_length': 1.5251970384523525, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5645052', 'n_tokens_mistral': 2699, 'n_tokens_neox': 2556, 'n_words': 764}