<commit_before>from future import standard_library
standard_library.install_aliases()
from builtins import next
from builtins import object

import csv
import io
import sys
from functools import reduce

from retriever.lib.cleanup import *


class Table(object):
    """Information about a database table."""

    def __init__(self, name, **kwargs):
        self.name = name
        self.pk = True
        self.contains_pk = False
        self.delimiter = None
        self.header_rows = 1
        self.column_names_row = 1
        self.fixed_width = False
        self.cleanup = Cleanup()
        self.record_id = 0
        self.columns = []
        self.replace_columns = []
        self.escape_single_quotes = True
        self.escape_double_quotes = True
        self.cleaned_columns = False
        for key, item in list(kwargs.items()):
            setattr(self, key, item[0] if isinstance(item, tuple) else item)

    def auto_get_columns(self, header):
        """Gets the column names from the header row

        Identifies the column names from the header row.
        Replaces database keywords with alternatives.
        Replaces special characters and spaces.
        """
        column_names = self.extract_values(header)
        columns = [self.clean_column_name(x) for x in column_names]
        column_values = {x: [] for x in columns if x}
        self.cleaned_columns = True
        return [[x, None] for x in columns if x], column_values

    def clean_column_name(self, column_name):
        """Clean column names using the expected sql guidelines
        remove leading whitespaces, replace sql key words, etc..
        """
        column_name = column_name.lower().strip()
        replace_columns = {old.lower(): new.lower()
                           for old, new in self.replace_columns}
        column_name = str(replace_columns.get(column_name, column_name).strip())
        replace = [
            ("%", "percent"),
            ("&", "and"),
            ("\xb0", "degrees"),
            ("?", ""),
            ("^", "_power_"),
            ("<", "_lthn_"),
            (">", "_gthn_"),
        ]
        replace += [(x, '') for x in (")", "\n", "\r", '"', "'")]
        replace += [(x, '_') for x in (" ", "(", "/", ".", "-", "*", ":", "[", "]")]

        column_name = reduce(lambda x, y: x.replace(*y), replace, column_name)

        while "__" in column_name:
            column_name = column_name.replace("__", "_")
        column_name = column_name.lstrip("0123456789_").rstrip("_")
        replace_dict = {
            "group": "grp",
            "order": "ordered",
            "check": "checked",
            "references": "refs",
            "long": "lon",
            "column": "columns",
            "cursor": "cursors",
            "delete": "deleted",
            "insert": "inserted",
            "join": "joins",
            "select": "selects",
            "table": "tables",
            "update": "updates",
            "date": "record_date"
        }
        for x in (")", "\n", "\r", '"', "'"):
            replace_dict[x] = ''
        for x in (" ", "(", "/", ".", "-"):<commit_msg>Add value to replace in column<commit_after>from future import standard_library
standard_library.install_aliases()
from builtins import next
from builtins import object

import csv
import io
import sys
from functools import reduce

from retriever.lib.cleanup import *


class Table(object):
    """Information about a database table."""

    def __init__(self, name, **kwargs):
        self.name = name
        self.pk = True
        self.contains_pk = False
        self.delimiter = None
        self.header_rows = 1
        self.column_names_row = 1
        self.fixed_width = False
        self.cleanup = Cleanup()
        self.record_id = 0
        self.columns = []
        self.replace_columns = []
        self.escape_single_quotes = True
        self.escape_double_quotes = True
        self.cleaned_columns = False
        for key, item in list(kwargs.items()):
            setattr(self, key, item[0] if isinstance(item, tuple) else item)

    def auto_get_columns(self, header):
        """Gets the column names from the header row

        Identifies the column names from the header row.
        Replaces database keywords with alternatives.
        Replaces special characters and spaces.
        """
        column_names = self.extract_values(header)
        columns = [self.clean_column_name(x) for x in column_names]
        column_values = {x: [] for x in columns if x}
        self.cleaned_columns = True
        return [[x, None] for x in columns if x], column_values

    def clean_column_name(self, column_name):
        """Clean column names using the expected sql guidelines
        remove leading whitespaces, replace sql key words, etc..
        """
        column_name = column_name.lower().strip()
        replace_columns = {old.lower(): new.lower()
                           for old, new in self.replace_columns}
        column_name = str(replace_columns.get(column_name, column_name).strip())
        replace = [
            ("%", "percent"),
            ("&", "and"),
            ("\xb0", "degrees"),
            ("?", ""),
            ("#", ""),
            ("^", "_power_"),
            ("<", "_lthn_"),
            (">", "_gthn_"),
        ]
        replace += [(x, '') for x in (")", "\n", "\r", '"', "'")]
        replace += [(x, '_') for x in (" ", "(", "/", ".", "-", "*", ":", "[", "]")]

        column_name = reduce(lambda x, y: x.replace(*y), replace, column_name)

        while "__" in column_name:
            column_name = column_name.replace("__", "_")
        column_name = column_name.lstrip("0123456789_").rstrip("_")
        replace_dict = {
            "group": "grp",
            "order": "ordered",
            "check": "checked",
            "references": "refs",
            "long": "lon",
            "column": "columns",
            "cursor": "cursors",
            "delete": "deleted",
            "insert": "inserted",
            "join": "joins",
            "select": "selects",
            "table": "tables",
            "update": "updates",
            "date": "record_date"
        }
        for x in (")", "\n", "\r", '"', "'"):
            replace_dict[x] = ''
        for x in (" ", "(", "/", ".", "-"):
{'fraction_non_alphanumeric': 0.1490003173595684, 'fraction_numerical': 0.004760393525864805, 'mean_word_length': 2.0157894736842104, 'pattern_counts': {'":': 30, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2583436', 'n_tokens_mistral': 1825, 'n_tokens_neox': 1777, 'n_words': 479}