<commit_before>"""
Management command to backfill history.
"""
import csv
import logging
import os
import time

from django.core.management.base import BaseCommand
from django.db import connection, transaction

log = logging.getLogger(__name__)


class Command(BaseCommand):
    """
    Backfill history for models using django-simple-history.
    This is a one-off and would be removed after it is executed.
    Example usage:
    $ ./manage.py backfill_history --batchsize 1000 --sleep_between 1 --settings=devstack
    """

    help = (
        "Populates the historical records with snapshot data."
    )

    DEFAULT_SIZE = 1000
    DEFAULT_SLEEP_BETWEEN_INSERTS = 1
    DATE = '2019-06-29'
    HISTORY_USER_ID = None
    HISTORY_CHANGE_REASON = 'initial history population'

    TABLES = [
        {'name': 'offer_benefit', 'exclude_column': None, 'input_filename': 'offer_benefit_2019-06-29'},
        {'name': 'offer_range', 'exclude_column': 'slug', 'input_filename': 'offer_range_2019-06-29'},
        {'name': 'offer_condition', 'exclude_column': None, 'input_filename': 'offer_condition_2019-06-29'},
        # pylint: disable=C0301
        {'name': 'offer_conditionaloffer', 'exclude_column': 'slug', 'input_filename': 'offer_conditionaloffer_2019-06-29'},
        # pylint: disable=C0301
        {'name': 'offer_offerassignment', 'exclude_column': 'slug', 'input_filename': 'offer_offerassignment_2019-06-29'},
        {'name': 'catalogue_product', 'exclude_column': None, 'input_filename': 'catalogue_product_2019-06-29'},
        # pylint: disable=C0301
        {'name': 'catalogue_productattributevalue', 'exclude_column': None, 'input_filename': 'catalogue_productattributevalue_2019-06-29'},
        {'name': 'order_order', 'exclude_column': None, 'input_filename': 'order_order_2019-06-29'},
        {'name': 'order_line', 'exclude_column': None, 'input_filename': 'order_line_2019-06-29'},
        {'name': 'partner_stockrecord', 'exclude_column': None, 'input_filename': 'partner_stockrecord_2019-06-29'},
        {'name': 'refund_refund', 'exclude_column': None, 'input_filename': 'refund_refund_2019-06-29'},
        {'name': 'refund_refundline', 'exclude_column': None, 'input_filename': 'refund_refundline_2019-06-29'},
        {'name': 'core_businessclient', 'exclude_column': None, 'input_filename': 'core_businessclient_2019-06-29'},
        {'name': 'courses_course', 'exclude_column': None, 'input_filename': 'courses_course_2019-06-29'},
    ]

    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)

        parser.add_argument(
            '--sleep_between',
            default=self.DEFAULT_SLEEP_BETWEEN_INSERTS,
            type=float,
            help='Seconds to sleep between chunked inserts.'
        )

        parser.add_argument(
            "--size",
            action="store",
            default=self.DEFAULT_SIZE,
            type=int,
            help="Maximum number of rows per insert.",
        )

        parser.add_argument(
            "--input_root",
            action="store",
        )

    def chunks(self, ids, chunk_size):
        for i in range(0, len(ids), chunk_size):
            yield ids[i:i + chunk_size]

    def replace_values(self, values, original, replacement):
        values = [[replacement if v == original else v for v in value] for value in values]
        return values

    def handle(self, *args, **options):
        batch_size = options['size']
        sleep_between = options['sleep_between']
        input_root = options['input_root']

        for table_info in self.TABLES:
            table = table_info['name']
            historical_table = "_historical".join(table.rsplit('_', 1))
            exclude_column = table_info['exclude_column']<commit_msg>fix example in history backfill command<commit_after>"""
Management command to backfill history.
"""
import csv
import logging
import os
import time

from django.core.management.base import BaseCommand
from django.db import connection, transaction

log = logging.getLogger(__name__)


class Command(BaseCommand):
    """
    Backfill history for models using django-simple-history.
    This is a one-off and would be removed after it is executed.
    Example usage:
    $ ./manage.py backfill_history --batch_size 1000 --sleep_between 1 --input_root /tmp/data/ --settings=devstack
    """

    help = (
        "Populates the historical records with snapshot data."
    )

    DEFAULT_SIZE = 1000
    DEFAULT_SLEEP_BETWEEN_INSERTS = 1
    DATE = '2019-06-29'
    HISTORY_USER_ID = None
    HISTORY_CHANGE_REASON = 'initial history population'

    TABLES = [
        {'name': 'offer_benefit', 'exclude_column': None, 'input_filename': 'offer_benefit_2019-06-29'},
        {'name': 'offer_range', 'exclude_column': 'slug', 'input_filename': 'offer_range_2019-06-29'},
        {'name': 'offer_condition', 'exclude_column': None, 'input_filename': 'offer_condition_2019-06-29'},
        # pylint: disable=C0301
        {'name': 'offer_conditionaloffer', 'exclude_column': 'slug', 'input_filename': 'offer_conditionaloffer_2019-06-29'},
        # pylint: disable=C0301
        {'name': 'offer_offerassignment', 'exclude_column': 'slug', 'input_filename': 'offer_offerassignment_2019-06-29'},
        {'name': 'catalogue_product', 'exclude_column': None, 'input_filename': 'catalogue_product_2019-06-29'},
        # pylint: disable=C0301
        {'name': 'catalogue_productattributevalue', 'exclude_column': None, 'input_filename': 'catalogue_productattributevalue_2019-06-29'},
        {'name': 'order_order', 'exclude_column': None, 'input_filename': 'order_order_2019-06-29'},
        {'name': 'order_line', 'exclude_column': None, 'input_filename': 'order_line_2019-06-29'},
        {'name': 'partner_stockrecord', 'exclude_column': None, 'input_filename': 'partner_stockrecord_2019-06-29'},
        {'name': 'refund_refund', 'exclude_column': None, 'input_filename': 'refund_refund_2019-06-29'},
        {'name': 'refund_refundline', 'exclude_column': None, 'input_filename': 'refund_refundline_2019-06-29'},
        {'name': 'core_businessclient', 'exclude_column': None, 'input_filename': 'core_businessclient_2019-06-29'},
        {'name': 'courses_course', 'exclude_column': None, 'input_filename': 'courses_course_2019-06-29'},
    ]

    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)

        parser.add_argument(
            '--sleep_between',
            default=self.DEFAULT_SLEEP_BETWEEN_INSERTS,
            type=float,
            help='Seconds to sleep between chunked inserts.'
        )

        parser.add_argument(
            "--batch_size",
            action="store",
            default=self.DEFAULT_SIZE,
            type=int,
            help="Maximum number of rows per insert.",
        )

        parser.add_argument(
            "--input_root",
            action="store",
            help="Path containing data files from snapshot for history backfill"
        )

    def chunks(self, ids, chunk_size):
        for i in range(0, len(ids), chunk_size):
            yield ids[i:i + chunk_size]

    def replace_values(self, values, original, replacement):
        values = [[replacement if v == original else v for v in value] for value in values]
        return values

    def handle(self, *args, **options):
        batch_size = options['batch_size']
        sleep_between = options['sleep_between']
        input_root = options['input_root']

        for table_info in self.TABLES:
            table = table_info['name']
            historical_table = "_historical".join(table.rsplit('_', 1))
            exclude_column = table_info['exclude_column']
{'fraction_non_alphanumeric': 0.13026521060842433, 'fraction_numerical': 0.0374414976599064, 'mean_word_length': 3.312219730941704, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5708800', 'n_tokens_mistral': 2550, 'n_tokens_neox': 2259, 'n_words': 515}