<commit_before>"""The django management command sync_from_asana"""
import logging

from asana.error import NotFoundError, InvalidTokenError, ForbiddenError
from django.apps import apps
from django.core.management.base import BaseCommand, CommandError
from django.utils import six

from djasana.connect import client_connect
from djasana.models import (
    Attachment, Project, Story, SyncToken,
    Tag, Task, Team, User, Webhook, Workspace)
from djasana.settings import settings
from djasana.utils import (
    pop_unsupported_fields, set_webhook, sync_attachment, sync_project, sync_story, sync_task)

logger = logging.getLogger(__name__)


class Command(BaseCommand):
    """Sync data from Asana to the database"""
    help = 'Import data from Asana and insert/update model instances'
    commit = True
    client = None
    process_archived = False
    synced_ids = []  # A running list of remote ids of tasks that have been synced.

    @staticmethod
    def get_client():
        return client_connect()

    def add_arguments(self, parser):
        parser.add_argument(
            '--noinput', action='store_false', dest='interactive', default=True,
            help='If provided, no prompts will be issued to the user and the data will be synced.'
        )
        parser.add_argument(
            "-w", "--workspace", action="append", default=[],
            help='Sync only the named workspace (can be used multiple times). '
                 'By default all workspaces will be updated from Asana.'
        )
        parser.add_argument(
            "-p", "--project", action="append", default=[],
            help='Sync only the named project (can be used multiple times). '
                 'By default all projects will be updated from Asana.'
        )
        parser.add_argument(
            "-m", "--model", action="append", default=[],
            help='Sync only the named model (can be used multiple times). '
                 'By default all models will be updated from Asana.'
        )
        parser.add_argument(
            "-mx", "--model-exclude", action="append", default=[],
            help='Exclude the named model (can be used multiple times).'
        )
        parser.add_argument(
            "-a", "--archive", action="store_false", dest='archive',
            help='Sync project tasks etc. even if the project is archived. '
                 'By default, only tasks of unarchived projects are updated from Asana. '
                 'Regardless of this setting, the project itself will be updated, '
                 'perhaps becoming marked as archived. '
        )
        parser.add_argument(
            '--nocommit', action='store_false', dest='commit',
            default=True, help='Will not commit changes to the database.'
        )

    def handle(self, *args, **options):
        self.commit = not options.get('nocommit')
        if self.commit and options.get('interactive', True):
            self.stdout.write(
                'WARNING: This will irreparably synchronize your local database from Asana.')
            if not self._confirm():
                self.stdout.write("No action taken.")
                return
        self.process_archived = options.get('archive')
        models = self._get_models(options)
        if options.get('verbosity', 0) >= 1:
            message = "Synchronizing data from Asana."
            self.stdout.write(message)
            logger.info(message)
        workspaces = options.get('workspace') or []
        if settings.ASANA_WORKSPACE:
            workspaces.append(settings.ASANA_WORKSPACE)
        # Allow client to be mocked:
        self.client = self.client or self.get_client()
        workspace_ids = self._get_workspace_ids(workspaces)
        projects = options.get('project')

        for workspace_id in workspace_ids:
            self._sync_workspace_id(workspace_id, projects, models)

    @staticmethod
    def _confirm():
        yes_or_no = six.moves.input("Are you sure you wish to continue? [y/N] ")
        return yes_or_no.lower().startswith('y')

    @staticmethod
    def _get_models(options):
        """Returns a list of models to sync"""
        models = options.get('model')
        models_exclude = options.get('model_exclude')
        app_models = list(apps.get_app_config('djasana').get_models())
        if models:
            good_models = []
            model_names = [model_.__name__.lower() for model_ in app_models]
            for model in models:
                try:
                    index = model_names.index(model.lower())
                except ValueError:
                    raise CommandError('{} is not an Asana model'.format(model))
                else:
                    good_models.append(app_models[index])
            models = good_models
        else:
            models = app_models
        if models_exclude:
            models = [model
                      for model in models
                      if model.__name__.lower() not in [m.lower() for m in models_exclude]]
        return models

    def _check_sync_project_id(self, project_id, workspace, models):
        """If we have a valid sync token for this project sync new events else sync the project"""
        new_sync = False
        try:
            sync_token = SyncToken.objects.get(project_id=project_id)
            try:
                events = self.client.events.get({'resource': project_id, 'sync': sync_token.sync})
                self._process_events(project_id, events, models)
                self._set_webhook(workspace, project_id)
                return
            except InvalidTokenError as error:
                sync_token.sync = error.sync
                sync_token.save()
        except SyncToken.DoesNotExist:
            try:
                self.client.events.get({'resource': project_id})
            except InvalidTokenError as error:
                new_sync = error.sync
        is_archived = self._sync_project_id(project_id, models)
        if not is_archived:
            self._set_webhook(workspace, project_id)
        if new_sync:
            SyncToken.objects.create(project_id=project_id, sync=new_sync)

    def _get_workspace_ids(self, workspaces):
        workspace_ids = []
        bad_list = []
        workspaces_ = self.client.workspaces.find_all()
        if workspaces:
            for workspace in workspaces:
                for wks in workspaces_:
                    if workspace in (wks['gid'], ['name']):
                        workspace_ids.append(wks['gid'])
                        break
                else:
                    bad_list.append(workspace)
        else:
            workspace_ids = [wks['gid'] for wks in workspaces_]
        if bad_list:
            if len(bad_list) == 1:
                raise CommandError('{} is not an Asana workspace'.format(workspaces[0]))
            raise CommandError('Specified workspaces are not valid: {}'.format(
                ', '.join(bad_list)))
        # Return newer workspaces first so they get synced earlier
        return sorted(workspace_ids, reverse=True)

    def _get_project_ids(self, projects, workspace_id):
        project_ids = []
        bad_list = []

        projects_ = self.client.projects.find_all({'workspace': workspace_id})
        if projects:
            for project in projects:
                for prj in projects_:
                    if project in (prj['gid'], prj['name']):
                        project_ids.append(prj['gid'])
                        break
                else:
                    bad_list.append(project)
        else:<commit_msg>Fix bad linting<commit_after>"""The django management command sync_from_asana"""
import logging

from asana.error import NotFoundError, InvalidTokenError, ForbiddenError
from django.apps import apps
from django.core.management.base import BaseCommand, CommandError
from django.utils import six

from djasana.connect import client_connect
from djasana.models import (
    Attachment, Project, Story, SyncToken,
    Tag, Task, Team, User, Webhook, Workspace)
from djasana.settings import settings
from djasana.utils import (
    pop_unsupported_fields, set_webhook, sync_attachment, sync_project, sync_story, sync_task)

logger = logging.getLogger(__name__)


class Command(BaseCommand):
    """Sync data from Asana to the database"""
    help = 'Import data from Asana and insert/update model instances'
    commit = True
    client = None
    process_archived = False
    synced_ids = []  # A running list of remote ids of tasks that have been synced.

    @staticmethod
    def get_client():
        return client_connect()

    def add_arguments(self, parser):
        parser.add_argument(
            '--noinput', action='store_false', dest='interactive', default=True,
            help='If provided, no prompts will be issued to the user and the data will be synced.'
        )
        parser.add_argument(
            "-w", "--workspace", action="append", default=[],
            help='Sync only the named workspace (can be used multiple times). '
                 'By default all workspaces will be updated from Asana.'
        )
        parser.add_argument(
            "-p", "--project", action="append", default=[],
            help='Sync only the named project (can be used multiple times). '
                 'By default all projects will be updated from Asana.'
        )
        parser.add_argument(
            "-m", "--model", action="append", default=[],
            help='Sync only the named model (can be used multiple times). '
                 'By default all models will be updated from Asana.'
        )
        parser.add_argument(
            "-mx", "--model-exclude", action="append", default=[],
            help='Exclude the named model (can be used multiple times).'
        )
        parser.add_argument(
            "-a", "--archive", action="store_false", dest='archive',
            help='Sync project tasks etc. even if the project is archived. '
                 'By default, only tasks of unarchived projects are updated from Asana. '
                 'Regardless of this setting, the project itself will be updated, '
                 'perhaps becoming marked as archived. '
        )
        parser.add_argument(
            '--nocommit', action='store_false', dest='commit',
            default=True, help='Will not commit changes to the database.'
        )

    def handle(self, *args, **options):
        self.commit = not options.get('nocommit')
        if self.commit and options.get('interactive', True):
            self.stdout.write(
                'WARNING: This will irreparably synchronize your local database from Asana.')
            if not self._confirm():
                self.stdout.write("No action taken.")
                return
        self.process_archived = options.get('archive')
        models = self._get_models(options)
        if options.get('verbosity', 0) >= 1:
            message = "Synchronizing data from Asana."
            self.stdout.write(message)
            logger.info(message)
        workspaces = options.get('workspace') or []
        if settings.ASANA_WORKSPACE:
            workspaces.append(settings.ASANA_WORKSPACE)
        # Allow client to be mocked:
        self.client = self.client or self.get_client()
        workspace_ids = self._get_workspace_ids(workspaces)
        projects = options.get('project')

        for workspace_id in workspace_ids:
            self._sync_workspace_id(workspace_id, projects, models)

    @staticmethod
    def _confirm():
        yes_or_no = six.moves.input("Are you sure you wish to continue? [y/N] ")
        return yes_or_no.lower().startswith('y')

    @staticmethod
    def _get_models(options):
        """Returns a list of models to sync"""
        models = options.get('model')
        models_exclude = options.get('model_exclude')
        app_models = list(apps.get_app_config('djasana').get_models())
        if models:
            good_models = []
            model_names = [model_.__name__.lower() for model_ in app_models]
            for model in models:
                try:
                    index = model_names.index(model.lower())
                except ValueError:
                    raise CommandError('{} is not an Asana model'.format(model))
                else:
                    good_models.append(app_models[index])
            models = good_models
        else:
            models = app_models
        if models_exclude:
            models = [model
                      for model in models
                      if model.__name__.lower() not in [m.lower() for m in models_exclude]]
        return models

    def _check_sync_project_id(self, project_id, workspace, models):
        """If we have a valid sync token for this project sync new events else sync the project"""
        new_sync = False
        try:
            sync_token = SyncToken.objects.get(project_id=project_id)
            try:
                events = self.client.events.get({'resource': project_id, 'sync': sync_token.sync})
                self._process_events(project_id, events, models)
                self._set_webhook(workspace, project_id)
                return
            except InvalidTokenError as error:
                sync_token.sync = error.sync
                sync_token.save()
        except SyncToken.DoesNotExist:
            try:
                self.client.events.get({'resource': project_id})
            except InvalidTokenError as error:
                new_sync = error.sync
        is_archived = self._sync_project_id(project_id, models)
        if not is_archived:
            self._set_webhook(workspace, project_id)
        if new_sync:
            SyncToken.objects.create(project_id=project_id, sync=new_sync)

    def _get_workspace_ids(self, workspaces):
        workspace_ids = []
        bad_list = []
        workspaces_ = self.client.workspaces.find_all()
        if workspaces:
            for workspace in workspaces:
                for wks in workspaces_:
                    if workspace in (wks['gid'], wks['name']):
                        workspace_ids.append(wks['gid'])
                        break
                else:
                    bad_list.append(workspace)
        else:
            workspace_ids = [wks['gid'] for wks in workspaces_]
        if bad_list:
            if len(bad_list) == 1:
                raise CommandError('{} is not an Asana workspace'.format(workspaces[0]))
            raise CommandError('Specified workspaces are not valid: {}'.format(
                ', '.join(bad_list)))
        # Return newer workspaces first so they get synced earlier
        return sorted(workspace_ids, reverse=True)

    def _get_project_ids(self, projects, workspace_id):
        project_ids = []
        bad_list = []

        projects_ = self.client.projects.find_all({'workspace': workspace_id})
        if projects:
            for project in projects:
                for prj in projects_:
                    if project in (prj['gid'], prj['name']):
                        project_ids.append(prj['gid'])
                        break
                else:
                    bad_list.append(project)
        else:
{'fraction_non_alphanumeric': 0.09368635437881874, 'fraction_numerical': 0.0005255896458839761, 'mean_word_length': 2.0437912417516495, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '969569', 'n_tokens_mistral': 4065, 'n_tokens_neox': 3922, 'n_words': 1157}