<commit_before>from future.standard_library import install_aliases
install_aliases()  # noqa
from urllib.parse import urlparse

from celery.task import Task
from django.conf import settings
from seed_services_client import IdentityStoreApiClient

from .models import Parish


class SyncLocations(Task):
    """
    Has a look at all the identity store identities, and ensures that all of
    the locations assigned to identities appear in the list of locations.
    """
    def get_identities(self, client):
        """
        Returns an iterator over all the identities in the identity store
        specified by 'client'.
        """
        identities = client.get_identities()
        while True:
            for identity in identities.get('results', []):
                yield identity
            if identities.get('next') is not None:
                qs = urlparse(identities['next']).query
                identities = client.get_identities(params=qs)
            else:
                break

    def run(self, **kwargs):
        l = self.get_logger(**kwargs)
        l.info('Starting location import')
        imported_count = 0
        client = IdentityStoreApiClient(<commit_msg>Add comment to explain pagination handling<commit_after>from future.standard_library import install_aliases
install_aliases()  # noqa
from urllib.parse import urlparse

from celery.task import Task
from django.conf import settings
from seed_services_client import IdentityStoreApiClient

from .models import Parish


class SyncLocations(Task):
    """
    Has a look at all the identity store identities, and ensures that all of
    the locations assigned to identities appear in the list of locations.
    """
    def get_identities(self, client):
        """
        Returns an iterator over all the identities in the identity store
        specified by 'client'.
        """
        identities = client.get_identities()
        while True:
            for identity in identities.get('results', []):
                yield identity
            # If there is a next page, extract the querystring and get it
            if identities.get('next') is not None:
                qs = urlparse(identities['next']).query
                identities = client.get_identities(params=qs)
            else:
                break

    def run(self, **kwargs):
        l = self.get_logger(**kwargs)
        l.info('Starting location import')
        imported_count = 0
        client = IdentityStoreApiClient(
{'fraction_non_alphanumeric': 0.07243460764587525, 'fraction_numerical': 0.0008048289738430583, 'mean_word_length': 2.516265912305516, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4695134', 'n_tokens_mistral': 639, 'n_tokens_neox': 604, 'n_words': 222}