<commit_before>import re
import json
from django.core.exceptions import ObjectDoesNotExist
from soupmigration.utils import regex_lookups, remove_lookup_type
from django.db import settings
import MySQLdb

__all__ = ['Data', 'Migration', 'Log']


class Data(object):
    """ Load tables and their data from a MySQL database.

    Explanation of optional class attributes:

    `data_filter` should be a dictionary of keys and values to determine if
     a dictionary ("row") should be deleted from `data`. Example:
        data_filter = {
            column1: 'not_used',
            ...
        }
        Will remove all dicts in `data` where key 'column1' has the value
        'not_used'.

    `mapping` is an optional but useful attribute that is used to specify
     tables and columns to fetch. Can also be used to rename columns. To get a
     default mapping, use the method `_get_mapping_keys`. Example mapping:
            mapping = {
                'all': {
                    column1: new_column_name1,
                    ...
                },
                table1: {
                    column2: new_column_name2,
                    column3: None, # Not renamed
                },
                table2: {
                    column4: new_column_name3,
                    ...
                }
                ...
            }

    `empty_values` is a list of strings that will be replaced with an empty
     string if found in `data`.

    `data` contains all loaded data and will look something like this:
        {
            table1: [
                {
                    column_name1: data1,
                    column_name2: data2,
                    ...
                },
                ...
            ]
            table2: ...
        }


    If `mapping` is defined the class attribute `merged_data` will contains
    the merged data of the tables in `data`. The following format is used:
        [
            {
                column_name1: data1,
                column_name2: data2,
                ...
            },
            ...
        ]
    """
    empty_values = []

    def __init__(self, **kwargs):
        self.connect_kwargs = {
            'host': getattr(settings, 'OLD_DB_HOST', '') or
                            kwargs.pop('host', ''),
            'user': getattr(settings, 'OLD_DB_USER', '') or
                            kwargs.pop('user', ''),
            'passwd': getattr(settings, 'OLD_DB_PASS', '') or
                              kwargs.pop('passwd', ''),
            'db': getattr(settings, 'OLD_DB_DATABASE', '') or
                          kwargs.pop('db', ''),
            'use_unicode': True,
            'charset': 'utf8',
        }
        self.connection = MySQLdb.connect(**self.connect_kwargs)
        self.cursor = self.connection.cursor()
        self.tables = []
        self.data = {}
        self.merged_data = []
        self.log = Log()
        self.empty_values += [None, 'None', 'NULL', '0']

        # Warn on invalid keyword arguments.
        for kwarg in kwargs:
            print '"{}" is not a valid keyword argument.'.format(kwarg)

        self.load_data()
        self.clean()

    def load_table_names(self):
        """ Load all table names """
        self.cursor.execute('SHOW TABLES')
        self.tables = [table[0] for table in self.cursor.fetchall()]
        if hasattr(self, 'mapping'):
            assert isinstance(self.mapping, dict), '`mapping` must be ' \
                'a dictionary.'
            assert set(self.mapping.keys()) <= set(self.tables), \
                'All keys in `mapping` be existing table names.'
            self.tables = [t for t in self.tables if t in self.mapping]

    def load_data(self):
        """ Load data from tables into `self.data` """

        if not self.tables:
            self.load_table_names()

        for table in self.tables:
            self.data[table] = []

            # Get column names
            self.cursor.execute('DESCRIBE `{}`'.format(table))
            keys = [column[0] for column in self.cursor.fetchall()]

            # Get data
            self.cursor.execute('SELECT * FROM `{}`'.format(table))
            all_data = [data for data in self.cursor.fetchall()]

            # Append data
            for row in all_data:
                assert len(keys) is len(row)
                dic = dict(zip(keys, row))

                # If we have a mapping, rename and delete keys as specified.
                if hasattr(self, 'mapping'):
                    mapping_keys = self._get_mapping_keys()
                    items_to_add = {}<commit_msg>Removed assert as keys in `mapping` must no longer be valid table names ('all' is a valid key).<commit_after>import re
import json
from django.core.exceptions import ObjectDoesNotExist
from soupmigration.utils import regex_lookups, remove_lookup_type
from django.db import settings
import MySQLdb

__all__ = ['Data', 'Migration', 'Log']


class Data(object):
    """ Load tables and their data from a MySQL database.

    Explanation of optional class attributes:

    `data_filter` should be a dictionary of keys and values to determine if
     a dictionary ("row") should be deleted from `data`. Example:
        data_filter = {
            column1: 'not_used',
            ...
        }
        Will remove all dicts in `data` where key 'column1' has the value
        'not_used'.

    `mapping` is an optional but useful attribute that is used to specify
     tables and columns to fetch. Can also be used to rename columns. To get a
     default mapping, use the method `_get_mapping_keys`. Example mapping:
            mapping = {
                'all': {
                    column1: new_column_name1,
                    ...
                },
                table1: {
                    column2: new_column_name2,
                    column3: None, # Not renamed
                },
                table2: {
                    column4: new_column_name3,
                    ...
                }
                ...
            }

    `empty_values` is a list of strings that will be replaced with an empty
     string if found in `data`.

    `data` contains all loaded data and will look something like this:
        {
            table1: [
                {
                    column_name1: data1,
                    column_name2: data2,
                    ...
                },
                ...
            ]
            table2: ...
        }


    If `mapping` is defined the class attribute `merged_data` will contains
    the merged data of the tables in `data`. The following format is used:
        [
            {
                column_name1: data1,
                column_name2: data2,
                ...
            },
            ...
        ]
    """
    empty_values = []

    def __init__(self, **kwargs):
        self.connect_kwargs = {
            'host': getattr(settings, 'OLD_DB_HOST', '') or
                            kwargs.pop('host', ''),
            'user': getattr(settings, 'OLD_DB_USER', '') or
                            kwargs.pop('user', ''),
            'passwd': getattr(settings, 'OLD_DB_PASS', '') or
                              kwargs.pop('passwd', ''),
            'db': getattr(settings, 'OLD_DB_DATABASE', '') or
                          kwargs.pop('db', ''),
            'use_unicode': True,
            'charset': 'utf8',
        }
        self.connection = MySQLdb.connect(**self.connect_kwargs)
        self.cursor = self.connection.cursor()
        self.tables = []
        self.data = {}
        self.merged_data = []
        self.log = Log()
        self.empty_values += [None, 'None', 'NULL', '0']

        # Warn on invalid keyword arguments.
        for kwarg in kwargs:
            print '"{}" is not a valid keyword argument.'.format(kwarg)

        self.load_data()
        self.clean()

    def load_table_names(self):
        """ Load all table names """
        self.cursor.execute('SHOW TABLES')
        self.tables = [table[0] for table in self.cursor.fetchall()]
        if hasattr(self, 'mapping'):
            assert isinstance(self.mapping, dict), '`mapping` must be ' \
                'a dictionary.'
            self.tables = [t for t in self.tables if t in self.mapping]

    def load_data(self):
        """ Load data from tables into `self.data` """

        if not self.tables:
            self.load_table_names()

        for table in self.tables:
            self.data[table] = []

            # Get column names
            self.cursor.execute('DESCRIBE `{}`'.format(table))
            keys = [column[0] for column in self.cursor.fetchall()]

            # Get data
            self.cursor.execute('SELECT * FROM `{}`'.format(table))
            all_data = [data for data in self.cursor.fetchall()]

            # Append data
            for row in all_data:
                assert len(keys) is len(row)
                dic = dict(zip(keys, row))

                # If we have a mapping, rename and delete keys as specified.
                if hasattr(self, 'mapping'):
                    mapping_keys = self._get_mapping_keys()
                    items_to_add = {}
{'fraction_non_alphanumeric': 0.10442073170731707, 'fraction_numerical': 0.00544425087108014, 'mean_word_length': 1.4993197278911565, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '76941', 'n_tokens_mistral': 2476, 'n_tokens_neox': 2370, 'n_words': 732}