<commit_before>import random
import requests
import shutil
import logging
import os
import traceback
import ujson

from typing import List, Dict, Any, Optional, Set, Callable, Iterable, Tuple, TypeVar
from django.forms.models import model_to_dict

from zerver.models import Realm, RealmEmoji, Subscription, Recipient, \
    Attachment, Stream, Message, UserProfile
from zerver.data_import.sequencer import NEXT_ID
from zerver.lib.actions import STREAM_ASSIGNMENT_COLORS as stream_colors
from zerver.lib.avatar_hash import user_avatar_path_from_ids
from zerver.lib.parallel import run_parallel

# stubs
ZerverFieldsT = Dict[str, Any]

class SubscriberHandler:
    def __init__(self) -> None:
        self.stream_info = dict()  # type: Dict[int, Set[int]]

    def set_info(self,
                 stream_id: int,
                 users: Set[int]) -> None:
        self.stream_info[stream_id] = users

    def get_users(self,
                  stream_id: int) -> Set[int]:
        users = self.stream_info[stream_id]
        return users

def build_zerver_realm(realm_id: int, realm_subdomain: str, time: float,
                       other_product: str) -> List[ZerverFieldsT]:
    realm = Realm(id=realm_id, date_created=time,
                  name=realm_subdomain, string_id=realm_subdomain,
                  description=("Organization imported from %s!" % (other_product)))
    auth_methods = [[flag[0], flag[1]] for flag in realm.authentication_methods]
    realm_dict = model_to_dict(realm, exclude='authentication_methods')
    realm_dict['authentication_methods'] = auth_methods
    return[realm_dict]

def build_user_profile(avatar_source: str,
                       date_joined: Any,
                       delivery_email: str,
                       email: str,
                       full_name: str,
                       id: int,
                       is_active: bool,
                       is_realm_admin: bool,
                       is_guest: bool,
                       is_mirror_dummy: bool,
                       realm_id: int,
                       short_name: str,
                       timezone: Optional[str]) -> ZerverFieldsT:
    pointer = -1
    obj = UserProfile(
        avatar_source=avatar_source,
        date_joined=date_joined,
        delivery_email=delivery_email,
        email=email,
        full_name=full_name,
        id=id,
        is_active=is_active,
        is_realm_admin=is_realm_admin,
        is_guest=is_guest,
        pointer=pointer,
        realm_id=realm_id,
        short_name=short_name,
        timezone=timezone,
    )
    dct = model_to_dict(obj)
    return dct

def build_avatar(zulip_user_id: int, realm_id: int, email: str, avatar_url: str,
                 timestamp: Any, avatar_list: List[ZerverFieldsT]) -> None:
    avatar = dict(
        path=avatar_url,  # Save original avatar url here, which is downloaded later
        realm_id=realm_id,
        content_type=None,
        user_profile_id=zulip_user_id,<commit_msg>import: Make use of is_mirror_dummy in build_user_profile.<commit_after>import random
import requests
import shutil
import logging
import os
import traceback
import ujson

from typing import List, Dict, Any, Optional, Set, Callable, Iterable, Tuple, TypeVar
from django.forms.models import model_to_dict

from zerver.models import Realm, RealmEmoji, Subscription, Recipient, \
    Attachment, Stream, Message, UserProfile
from zerver.data_import.sequencer import NEXT_ID
from zerver.lib.actions import STREAM_ASSIGNMENT_COLORS as stream_colors
from zerver.lib.avatar_hash import user_avatar_path_from_ids
from zerver.lib.parallel import run_parallel

# stubs
ZerverFieldsT = Dict[str, Any]

class SubscriberHandler:
    def __init__(self) -> None:
        self.stream_info = dict()  # type: Dict[int, Set[int]]

    def set_info(self,
                 stream_id: int,
                 users: Set[int]) -> None:
        self.stream_info[stream_id] = users

    def get_users(self,
                  stream_id: int) -> Set[int]:
        users = self.stream_info[stream_id]
        return users

def build_zerver_realm(realm_id: int, realm_subdomain: str, time: float,
                       other_product: str) -> List[ZerverFieldsT]:
    realm = Realm(id=realm_id, date_created=time,
                  name=realm_subdomain, string_id=realm_subdomain,
                  description=("Organization imported from %s!" % (other_product)))
    auth_methods = [[flag[0], flag[1]] for flag in realm.authentication_methods]
    realm_dict = model_to_dict(realm, exclude='authentication_methods')
    realm_dict['authentication_methods'] = auth_methods
    return[realm_dict]

def build_user_profile(avatar_source: str,
                       date_joined: Any,
                       delivery_email: str,
                       email: str,
                       full_name: str,
                       id: int,
                       is_active: bool,
                       is_realm_admin: bool,
                       is_guest: bool,
                       is_mirror_dummy: bool,
                       realm_id: int,
                       short_name: str,
                       timezone: Optional[str]) -> ZerverFieldsT:
    pointer = -1
    obj = UserProfile(
        avatar_source=avatar_source,
        date_joined=date_joined,
        delivery_email=delivery_email,
        email=email,
        full_name=full_name,
        id=id,
        is_mirror_dummy=is_mirror_dummy,
        is_active=is_active,
        is_realm_admin=is_realm_admin,
        is_guest=is_guest,
        pointer=pointer,
        realm_id=realm_id,
        short_name=short_name,
        timezone=timezone,
    )
    dct = model_to_dict(obj)
    return dct

def build_avatar(zulip_user_id: int, realm_id: int, email: str, avatar_url: str,
                 timestamp: Any, avatar_list: List[ZerverFieldsT]) -> None:
    avatar = dict(
        path=avatar_url,  # Save original avatar url here, which is downloaded later
        realm_id=realm_id,
        content_type=None,
        user_profile_id=zulip_user_id,
{'fraction_non_alphanumeric': 0.07839262187088274, 'fraction_numerical': 0.0009881422924901185, 'mean_word_length': 2.375764313507504, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4649231', 'n_tokens_mistral': 1889, 'n_tokens_neox': 1844, 'n_words': 410}