<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function

import csv
import getopt
import io
import itertools
import logging
import os
import re
import sqlite3
import string
import sys
import textwrap
import time

try:
    import readline
except ImportError:
    pass


PYTHON_3 = sys.version_info >= (3, )
EXIT_GENERAL_FAILURE = 1
EXIT_DATABASE_ERROR = 2

__all__ = ["PYTHON_3", "EXIT_GENERAL_FAILURE", "EXIT_DATABASE_ERROR",
    "SQLite3CSVImporter", "pretty_print_table", "query_split",
    "metaquery_conversion", "sqlite3_repl"]
__license__ = "BSD 2-Clause"


class SQLite3CSVImporter:
    sniffer = csv.Sniffer()
    typemap = [
        ("INTEGER", int),
        ("REAL", float),
        ("TEXT", (lambda v: v.encode("utf-8")) if PYTHON_3 else unicode),
        ("BLOB", (lambda v: v.encode("utf-8", errors="surrogateescape"))
                  if PYTHON_3 else str),
    ]

    def __init__(self, dbc, ignore_errors=True, log_warnings=True):
        """
        Setup SQLite3CSVImporter. When `ignore_errors` is set, any SQL errors
        encountered while inserting rows into the database will be ignored and,
        if `log_warnings` is set, a warning containing information about the
        failed INSERT will be logged.
        """
        self.dbc = dbc
        self.ignore_errors = ignore_errors
        self.log_warnings = log_warnings

    @classmethod
    def detect_types(cls, table):
        """
        Return list of SQL type definition clauses that can safely be applied
        to each of the columns in the `table`.
        """
        typedefs = list()
        for column in zip(*table):
            rows_with_content = [value for value in column if value]
            if not rows_with_content:
                scanned_columns = ("", )
            else:
                scanned_columns = rows_with_content

            for typedef, caster in cls.typemap:
                try:
                    for value in scanned_columns:
                        caster(value)
                    colschema = typedef
                    break
                except Exception:
                    pass
            else:
                raise ValueError("Could not detect type of %r" % (column,))

            typedefs.append(typedef)

        return typedefs

    @classmethod
    def quote_identifier(cls, identifier):
        """
        Return ANSI-quoted SQL identifier.
        """
        return '"' + identifier.replace('"', '""') + '"'

    def create_table(self, tablename, types, columns=None, if_not_exists=True):<commit_msg>Made quote_identifier a static method<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function

import csv
import getopt
import io
import itertools
import logging
import os
import re
import sqlite3
import string
import sys
import textwrap
import time

try:
    import readline
except ImportError:
    pass


PYTHON_3 = sys.version_info >= (3, )
EXIT_GENERAL_FAILURE = 1
EXIT_DATABASE_ERROR = 2

__all__ = ["PYTHON_3", "EXIT_GENERAL_FAILURE", "EXIT_DATABASE_ERROR",
    "SQLite3CSVImporter", "pretty_print_table", "query_split",
    "metaquery_conversion", "sqlite3_repl"]
__license__ = "BSD 2-Clause"


class SQLite3CSVImporter:
    sniffer = csv.Sniffer()
    typemap = [
        ("INTEGER", int),
        ("REAL", float),
        ("TEXT", (lambda v: v.encode("utf-8")) if PYTHON_3 else unicode),
        ("BLOB", (lambda v: v.encode("utf-8", errors="surrogateescape"))
                  if PYTHON_3 else str),
    ]

    def __init__(self, dbc, ignore_errors=True, log_warnings=True):
        """
        Setup SQLite3CSVImporter. When `ignore_errors` is set, any SQL errors
        encountered while inserting rows into the database will be ignored and,
        if `log_warnings` is set, a warning containing information about the
        failed INSERT will be logged.
        """
        self.dbc = dbc
        self.ignore_errors = ignore_errors
        self.log_warnings = log_warnings

    @classmethod
    def detect_types(cls, table):
        """
        Return list of SQL type definition clauses that can safely be applied
        to each of the columns in the `table`.
        """
        typedefs = list()
        for column in zip(*table):
            rows_with_content = [value for value in column if value]
            if not rows_with_content:
                scanned_columns = ("", )
            else:
                scanned_columns = rows_with_content

            for typedef, caster in cls.typemap:
                try:
                    for value in scanned_columns:
                        caster(value)
                    colschema = typedef
                    break
                except Exception:
                    pass
            else:
                raise ValueError("Could not detect type of %r" % (column,))

            typedefs.append(typedef)

        return typedefs

    @staticmethod
    def quote_identifier(identifier):
        """
        Return ANSI-quoted SQL identifier.
        """
        return '"' + identifier.replace('"', '""') + '"'

    def create_table(self, tablename, types, columns=None, if_not_exists=True):
{'fraction_non_alphanumeric': 0.09089162182936203, 'fraction_numerical': 0.006149116064565719, 'mean_word_length': 2.197174447174447, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3771764', 'n_tokens_mistral': 1536, 'n_tokens_neox': 1478, 'n_words': 440}