<commit_before>from __future__ import with_statement

from optparse import make_option

from django.conf import settings
from django.core.files.storage import get_storage_class
from django.core.management.base import CommandError, NoArgsCommand
from django.utils.encoding import smart_str

from ...utils import get_files


class Command(NoArgsCommand):
    """
    Command that allows to copy or symlink static files from different
    locations to the settings.STATIC_ROOT.
    """
    option_list = NoArgsCommand.option_list + (
        make_option('--noinput',
            action='store_false', dest='interactive', default=True,
            help="Do NOT prompt the user for input of any kind."),
        make_option('-i', '--ignore', action='append', default=[],
            dest='ignore_patterns', metavar='PATTERN',
            help="Ignore files or directories matching this glob-style "
                "pattern. Use multiple times to ignore more."),
        make_option('-n', '--dry-run',
            action='store_true', dest='dry_run', default=False,
            help="Do everything except modify the filesystem."),
        make_option('--no-default-ignore', action='store_false',
            dest='use_default_ignore_patterns', default=True,
            help="Don't ignore the common private glob-style patterns 'CVS', "
                "'.*' and '*~'."),
    )
    help = "Upload static files to S3."
    requires_model_validation = False

    def __init__(self, *args, **kwargs):
        super(NoArgsCommand, self).__init__(*args, **kwargs)
        self.uploaded_files = []
        self.skipped_files = []
        self.local_storage = get_storage_class('django.core.files.storage.FileSystemStorage')(
            location=getattr(settings, 'STATIC_ROOT', None),
            base_url=getattr(settings, 'STATIC_URL', None))
        self.remote_storage = get_storage_class('storages.backends.s3boto.S3BotoStorage')()

    def set_options(self, **options):
        """
        Set instance variables based on an options dict
        """
        self.interactive = options['interactive']
        self.verbosity = int(options.get('verbosity', 1))
        self.dry_run = options['dry_run']
        ignore_patterns = options['ignore_patterns']
        ignore_patterns.extend(getattr(settings, 'UPLOADSTATIC_IGNORE_PATTERNS', []))
        if options['use_default_ignore_patterns']:
            ignore_patterns += ['CVS', '.*', '*~']
        self.ignore_patterns = list(set(ignore_patterns))

    def upload_file(self, path):
        if self.dry_run:
            self.log(u"Pretending to upload '%s'" % path, level=1)
        else:
            if self.remote_storage.exists(path):
                local_created = self.local_storage.created_time(path)
                local_size = self.local_storage.size(path)
                remote_created = self.remote_storage.created_time(path)
                remote_size = self.remote_storage.size(path)
                if remote_created >= local_created and remote_size == local_size:
                    self.log(u"Skipping '%s'" % path, level=2)
                    self.skipped_files.append(path)<commit_msg>Revert "Use created time instead of modified time."<commit_after>from __future__ import with_statement

from optparse import make_option

from django.conf import settings
from django.core.files.storage import get_storage_class
from django.core.management.base import CommandError, NoArgsCommand
from django.utils.encoding import smart_str

from ...utils import get_files


class Command(NoArgsCommand):
    """
    Command that allows to copy or symlink static files from different
    locations to the settings.STATIC_ROOT.
    """
    option_list = NoArgsCommand.option_list + (
        make_option('--noinput',
            action='store_false', dest='interactive', default=True,
            help="Do NOT prompt the user for input of any kind."),
        make_option('-i', '--ignore', action='append', default=[],
            dest='ignore_patterns', metavar='PATTERN',
            help="Ignore files or directories matching this glob-style "
                "pattern. Use multiple times to ignore more."),
        make_option('-n', '--dry-run',
            action='store_true', dest='dry_run', default=False,
            help="Do everything except modify the filesystem."),
        make_option('--no-default-ignore', action='store_false',
            dest='use_default_ignore_patterns', default=True,
            help="Don't ignore the common private glob-style patterns 'CVS', "
                "'.*' and '*~'."),
    )
    help = "Upload static files to S3."
    requires_model_validation = False

    def __init__(self, *args, **kwargs):
        super(NoArgsCommand, self).__init__(*args, **kwargs)
        self.uploaded_files = []
        self.skipped_files = []
        self.local_storage = get_storage_class('django.core.files.storage.FileSystemStorage')(
            location=getattr(settings, 'STATIC_ROOT', None),
            base_url=getattr(settings, 'STATIC_URL', None))
        self.remote_storage = get_storage_class('storages.backends.s3boto.S3BotoStorage')()

    def set_options(self, **options):
        """
        Set instance variables based on an options dict
        """
        self.interactive = options['interactive']
        self.verbosity = int(options.get('verbosity', 1))
        self.dry_run = options['dry_run']
        ignore_patterns = options['ignore_patterns']
        ignore_patterns.extend(getattr(settings, 'UPLOADSTATIC_IGNORE_PATTERNS', []))
        if options['use_default_ignore_patterns']:
            ignore_patterns += ['CVS', '.*', '*~']
        self.ignore_patterns = list(set(ignore_patterns))

    def upload_file(self, path):
        if self.dry_run:
            self.log(u"Pretending to upload '%s'" % path, level=1)
        else:
            if self.remote_storage.exists(path):
                local_modified = self.local_storage.modified_time(path)
                local_size = self.local_storage.size(path)
                remote_modified = self.remote_storage.modified_time(path)
                remote_size = self.remote_storage.size(path)
                if remote_modified >= local_modified and remote_size == local_size:
                    self.log(u"Skipping '%s'" % path, level=2)
                    self.skipped_files.append(path)
{'fraction_non_alphanumeric': 0.11539672544080605, 'fraction_numerical': 0.001889168765743073, 'mean_word_length': 2.8110377924415118, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2267476', 'n_tokens_mistral': 1816, 'n_tokens_neox': 1756, 'n_words': 394}