<commit_before>import datetime
import fnmatch
import mimetypes
import optparse
import os
import pyrax
import re
import swiftclient

from django.conf import settings
from django.core.management import call_command
from django.core.management.base import CommandError, NoArgsCommand

from cumulus.settings import CUMULUS
from cumulus.storage import sync_headers


class Command(NoArgsCommand):
    help = "Synchronizes static media to cloud files."
    option_list = NoArgsCommand.option_list + (
        optparse.make_option("-i", "--include", action="append", default=[],
                             dest="includes", metavar="PATTERN",
                             help="Include file or directories matching this glob-style "
                                  "pattern. Use multiple times to include more."),
        optparse.make_option("-e", "--exclude", action="append", default=[],
                             dest="excludes", metavar="PATTERN",
                             help="Exclude files or directories matching this glob-style "
                                  "pattern. Use multiple times to exclude more."),
        optparse.make_option("-w", "--wipe",
                             action="store_true", dest="wipe", default=False,
                             help="Wipes out entire contents of container first."),
        optparse.make_option("-t", "--test-run",
                             action="store_true", dest="test_run", default=False,
                             help="Performs a test run of the sync."),
        optparse.make_option("-q", "--quiet",
                             action="store_true", dest="test_run", default=False,
                             help="Do not display any output."),
        optparse.make_option("-c", "--container",
                             dest="container", help="Override STATIC_CONTAINER."),
    )

    def set_options(self, options):
        """
        Sets instance variables based on an options dict
        """
        # COMMAND LINE OPTIONS
        self.wipe = options.get("wipe")
        self.test_run = options.get("test_run")
        self.quiet = options.get("test_run")
        self.container_name = options.get("container")
        self.verbosity = int(options.get("verbosity"))
        if self.test_run:
            self.verbosity = 2
        cli_includes = options.get("includes")
        cli_excludes = options.get("excludes")

        # CUMULUS CONNECTION AND SETTINGS FROM SETTINGS.PY
        if not self.container_name:
            self.container_name = CUMULUS["STATIC_CONTAINER"]
        settings_includes = CUMULUS["INCLUDE_LIST"]
        settings_excludes = CUMULUS["EXCLUDE_LIST"]

        # PATH SETTINGS
        self.static_root = os.path.abspath(settings.STATIC_ROOT)
        self.static_url = settings.STATIC_URL
        if not self.static_root.endswith("/"):
            self.static_root = self.static_root + "/"
        if self.static_url.startswith("/"):
            self.static_url = self.static_url[1:]

        # SYNCSTATIC VARS
        # combine includes and excludes from the cli and django settings file
        self.includes = list(set(cli_includes + settings_includes))
        self.excludes = list(set(cli_excludes + settings_excludes))
        # transform glob patterns to regular expressions
        self.local_filenames = []
        self.create_count = 0
        self.upload_count = 0
        self.update_count = 0
        self.skip_count = 0
        self.delete_count = 0

    def connect_container(self):
        """
        Connects to a container using the swiftclient api.

        The container will be created and/or made public using the
        pyrax api if not already so.
        """
        self.conn = swiftclient.Connection(authurl=CUMULUS["AUTH_URL"],
                                           user=CUMULUS["USERNAME"],
                                           key=CUMULUS["API_KEY"],
                                           snet=CUMULUS["SERVICENET"],
                                           auth_version=CUMULUS["AUTH_VERSION"],
                                           tenant_name=CUMULUS["AUTH_TENANT_NAME"])
        try:
            self.conn.head_container(self.container_name)
        except swiftclient.client.ClientException as exception:
            if exception.msg == "Container HEAD failed":
                call_command("container_create", self.container_name)
            else:
                raise

        if CUMULUS["USE_PYRAX"]:
            public = not CUMULUS["SERVICENET"]
            pyrax.set_credentials(CUMULUS["USERNAME"], CUMULUS["API_KEY"])
            connection = pyrax.connect_to_cloudfiles(region=CUMULUS["REGION"],
                                                     public=public)
            container = connection.get_container(self.container_name)
            if not container.cdn_enabled:
                container.make_public(ttl=CUMULUS["TTL"])
        else:
            headers = {"X-Container-Read": ".r:*"}
            self.conn.post_container(self.container_name, headers=headers)

        self.container = self.conn.get_container(self.container_name)

    def handle_noargs(self, *args, **options):
        # setup
        self.set_options(options)
        self.connect_container()

        # wipe first<commit_msg>Use PYRAX_IDENTITY_TYPE for syncstatic when available.<commit_after>import datetime
import fnmatch
import mimetypes
import optparse
import os
import pyrax
import re
import swiftclient

from django.conf import settings
from django.core.management import call_command
from django.core.management.base import CommandError, NoArgsCommand

from cumulus.settings import CUMULUS
from cumulus.storage import sync_headers


class Command(NoArgsCommand):
    help = "Synchronizes static media to cloud files."
    option_list = NoArgsCommand.option_list + (
        optparse.make_option("-i", "--include", action="append", default=[],
                             dest="includes", metavar="PATTERN",
                             help="Include file or directories matching this glob-style "
                                  "pattern. Use multiple times to include more."),
        optparse.make_option("-e", "--exclude", action="append", default=[],
                             dest="excludes", metavar="PATTERN",
                             help="Exclude files or directories matching this glob-style "
                                  "pattern. Use multiple times to exclude more."),
        optparse.make_option("-w", "--wipe",
                             action="store_true", dest="wipe", default=False,
                             help="Wipes out entire contents of container first."),
        optparse.make_option("-t", "--test-run",
                             action="store_true", dest="test_run", default=False,
                             help="Performs a test run of the sync."),
        optparse.make_option("-q", "--quiet",
                             action="store_true", dest="test_run", default=False,
                             help="Do not display any output."),
        optparse.make_option("-c", "--container",
                             dest="container", help="Override STATIC_CONTAINER."),
    )

    def set_options(self, options):
        """
        Sets instance variables based on an options dict
        """
        # COMMAND LINE OPTIONS
        self.wipe = options.get("wipe")
        self.test_run = options.get("test_run")
        self.quiet = options.get("test_run")
        self.container_name = options.get("container")
        self.verbosity = int(options.get("verbosity"))
        if self.test_run:
            self.verbosity = 2
        cli_includes = options.get("includes")
        cli_excludes = options.get("excludes")

        # CUMULUS CONNECTION AND SETTINGS FROM SETTINGS.PY
        if not self.container_name:
            self.container_name = CUMULUS["STATIC_CONTAINER"]
        settings_includes = CUMULUS["INCLUDE_LIST"]
        settings_excludes = CUMULUS["EXCLUDE_LIST"]

        # PATH SETTINGS
        self.static_root = os.path.abspath(settings.STATIC_ROOT)
        self.static_url = settings.STATIC_URL
        if not self.static_root.endswith("/"):
            self.static_root = self.static_root + "/"
        if self.static_url.startswith("/"):
            self.static_url = self.static_url[1:]

        # SYNCSTATIC VARS
        # combine includes and excludes from the cli and django settings file
        self.includes = list(set(cli_includes + settings_includes))
        self.excludes = list(set(cli_excludes + settings_excludes))
        # transform glob patterns to regular expressions
        self.local_filenames = []
        self.create_count = 0
        self.upload_count = 0
        self.update_count = 0
        self.skip_count = 0
        self.delete_count = 0

    def connect_container(self):
        """
        Connects to a container using the swiftclient api.

        The container will be created and/or made public using the
        pyrax api if not already so.
        """
        self.conn = swiftclient.Connection(authurl=CUMULUS["AUTH_URL"],
                                           user=CUMULUS["USERNAME"],
                                           key=CUMULUS["API_KEY"],
                                           snet=CUMULUS["SERVICENET"],
                                           auth_version=CUMULUS["AUTH_VERSION"],
                                           tenant_name=CUMULUS["AUTH_TENANT_NAME"])
        try:
            self.conn.head_container(self.container_name)
        except swiftclient.client.ClientException as exception:
            if exception.msg == "Container HEAD failed":
                call_command("container_create", self.container_name)
            else:
                raise

        if CUMULUS["USE_PYRAX"]:
            public = not CUMULUS["SERVICENET"]
            if CUMULUS["PYRAX_IDENTITY_TYPE"]:
                pyrax.set_setting("identity_type", CUMULUS["PYRAX_IDENTITY_TYPE"])
            pyrax.set_credentials(CUMULUS["USERNAME"], CUMULUS["API_KEY"])
            connection = pyrax.connect_to_cloudfiles(region=CUMULUS["REGION"],
                                                     public=public)
            container = connection.get_container(self.container_name)
            if not container.cdn_enabled:
                container.make_public(ttl=CUMULUS["TTL"])
        else:
            headers = {"X-Container-Read": ".r:*"}
            self.conn.post_container(self.container_name, headers=headers)

        self.container = self.conn.get_container(self.container_name)

    def handle_noargs(self, *args, **options):
        # setup
        self.set_options(options)
        self.connect_container()

        # wipe first
{'fraction_non_alphanumeric': 0.09843068065744266, 'fraction_numerical': 0.0013000278577398087, 'mean_word_length': 1.9442318206670313, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3420871', 'n_tokens_mistral': 2880, 'n_tokens_neox': 2765, 'n_words': 620}