<commit_before>import random
from django.conf import settings
from django_countries import countries

from account.models import User, UserProfile
from agency.models import (
    OtherAgency,
    Agency,
    AgencyProfile,
    AgencyOffice,
    AgencyMember,
)
from agency.roles import AgencyRole
from common.consts import (
    PARTNER_TYPES,
)
from common.factories import (
    PartnerFactory,
    PartnerMemberFactory,
    AgencyOfficeFactory,
    OtherAgencyFactory,
    AgencyMemberFactory,
    OpenEOIFactory,
    UnsolicitedFactory,
    PartnerVerificationFactory,
    PartnerFlagFactory,
    UserFactory,
    get_partner_name,
    DirectEOIFactory,
)
from common.models import (
    AdminLevel1,
    Point,
    Sector,
    Specialization,
    CommonFile,
)
from notification.models import (
    Notification,
    NotifiedUser,
)
from partner.models import (
    Partner,
    PartnerProfile,
    PartnerMailingAddress,
    PartnerHeadOrganization,
    PartnerDirector,
    PartnerAuthorisedOfficer,
    PartnerPolicyArea,
    PartnerAuditAssessment,
    PartnerReporting,
    PartnerMandateMission,
    PartnerExperience,
    PartnerInternalControl,
    PartnerBudget,
    PartnerFunding,
    PartnerCollaborationPartnership,
    PartnerCollaborationEvidence,
    PartnerOtherInfo,
    PartnerMember,
    PartnerReview,
)
from partner.roles import PartnerRole
from project.models import (
    EOI,
    Pin,
    Application,
    ApplicationFeedback,
    Assessment,
)
from review.models import (
    PartnerFlag,
    PartnerVerification,
)
from sanctionslist.models import (
    SanctionedName,
    SanctionedNameMatch,
)


def clean_up_data_in_db():
    models_to_wipe = [
        User,
        UserProfile,

        OtherAgency,
        Agency,
        AgencyProfile,
        AgencyOffice,
        AgencyMember,

        AdminLevel1,
        Point,
        Sector,
        Specialization,
        CommonFile,

        Notification,
        NotifiedUser,

        Partner,
        PartnerProfile,
        PartnerMailingAddress,
        PartnerHeadOrganization,
        PartnerDirector,
        PartnerAuthorisedOfficer,
        PartnerPolicyArea,
        PartnerAuditAssessment,
        PartnerReporting,
        PartnerMandateMission,
        PartnerExperience,
        PartnerInternalControl,
        PartnerBudget,
        PartnerFunding,
        PartnerCollaborationPartnership,
        PartnerCollaborationEvidence,
        PartnerOtherInfo,
        PartnerMember,
        PartnerReview,

        EOI,
        Pin,
        Application,
        ApplicationFeedback,
        Assessment,

        PartnerFlag,
        PartnerVerification,

        SanctionedName,
        SanctionedNameMatch,
    ]

    if not settings.IS_PROD:
        print("Start DB Wipe")

        for model in models_to_wipe:
            print(f'Deleting {model.objects.all().count()} {model} instances')
            model.objects.all().delete()

        print("End DB wipe")


USERNAME_AGENCY_ROLE_POSTFIXES = {
    AgencyRole.ADMINISTRATOR.name: 'admin',
    AgencyRole.HQ_EDITOR.name: 'editor-hq',
    AgencyRole.READER.name: 'reader',
    AgencyRole.EDITOR_BASIC.name: 'editor',
    AgencyRole.EDITOR_ADVANCED.name: 'editor-adv',
    AgencyRole.PAM_USER.name: 'pam',
    AgencyRole.MFT_USER.name: 'mft',
}


def generate_fake_data(country_count=3):
    admin, created = User.objects.get_or_create(fullname='admin', defaults={
        'email': '<EMAIL>',
        'is_superuser': True,
        'is_staff': True,
    })
    password = '<PASSWORD>!'
    admin.set_password(password)
    admin.save()
    print("Superuser {}: {}/{}".format("created" if created else "updated", admin.email, password))

    agencies = [
        Agency.objects.get(name="UNICEF"),
        Agency.objects.get(name="WFP"),
        Agency.objects.get(name="UNHCR"),
    ]

    chosen_countries = random.choices(countries, k=country_count)

    OtherAgencyFactory()

    partner_count = 2
    ingo_hqs = [
        PartnerFactory(display_type=PARTNER_TYPES.international) for _ in range(partner_count)
    ]
    for index, hq in enumerate(ingo_hqs):
        PartnerVerificationFactory(partner=hq)
        for role_code, display_name in PartnerRole.get_choices():
            user = UserFactory(
                email=f'ingo-hq-{index}-{role_code.lower()}<EMAIL>'
            )
            PartnerMemberFactory(user=user, role=role_code, partner=hq)
            print(f'Created {user}')

    standard_partners_created = 0
    ingo_partners_created = 0
    for country_code, country_name in chosen_countries:
        for index in range(partner_count):
            for partner_type, display_type in PARTNER_TYPES:
                if partner_type == PARTNER_TYPES.international:
                    name_parts = ingo_hqs[index].legal_name.split(' ') + [country_code]
                    partner_kwargs = {
                        'legal_name': f'{"-".join(name_parts)}.{partner_type}'.lower(),
                        'hq': ingo_hqs[index]
                    }
                    ingo_partners_created += 1
                else:
                    def get_legal_name():
                        name_parts = get_partner_name().split(" ")
                        return f'{"-".join(name_parts)}.{partner_type}'.lower()

                    # Soft unique check
                    legal_name = get_legal_name()
                    while Partner.objects.filter(legal_name=legal_name).exists():
                        legal_name = get_legal_name()

                    partner_kwargs = {
                        'hq': None,
                        'legal_name': legal_name,
                    }<commit_msg>fix fkaedata indexing<commit_after>import random
from django.conf import settings
from django_countries import countries

from account.models import User, UserProfile
from agency.models import (
    OtherAgency,
    Agency,
    AgencyProfile,
    AgencyOffice,
    AgencyMember,
)
from agency.roles import AgencyRole
from common.consts import (
    PARTNER_TYPES,
)
from common.factories import (
    PartnerFactory,
    PartnerMemberFactory,
    AgencyOfficeFactory,
    OtherAgencyFactory,
    AgencyMemberFactory,
    OpenEOIFactory,
    UnsolicitedFactory,
    PartnerVerificationFactory,
    PartnerFlagFactory,
    UserFactory,
    get_partner_name,
    DirectEOIFactory,
)
from common.models import (
    AdminLevel1,
    Point,
    Sector,
    Specialization,
    CommonFile,
)
from notification.models import (
    Notification,
    NotifiedUser,
)
from partner.models import (
    Partner,
    PartnerProfile,
    PartnerMailingAddress,
    PartnerHeadOrganization,
    PartnerDirector,
    PartnerAuthorisedOfficer,
    PartnerPolicyArea,
    PartnerAuditAssessment,
    PartnerReporting,
    PartnerMandateMission,
    PartnerExperience,
    PartnerInternalControl,
    PartnerBudget,
    PartnerFunding,
    PartnerCollaborationPartnership,
    PartnerCollaborationEvidence,
    PartnerOtherInfo,
    PartnerMember,
    PartnerReview,
)
from partner.roles import PartnerRole
from project.models import (
    EOI,
    Pin,
    Application,
    ApplicationFeedback,
    Assessment,
)
from review.models import (
    PartnerFlag,
    PartnerVerification,
)
from sanctionslist.models import (
    SanctionedName,
    SanctionedNameMatch,
)


def clean_up_data_in_db():
    models_to_wipe = [
        User,
        UserProfile,

        OtherAgency,
        Agency,
        AgencyProfile,
        AgencyOffice,
        AgencyMember,

        AdminLevel1,
        Point,
        Sector,
        Specialization,
        CommonFile,

        Notification,
        NotifiedUser,

        Partner,
        PartnerProfile,
        PartnerMailingAddress,
        PartnerHeadOrganization,
        PartnerDirector,
        PartnerAuthorisedOfficer,
        PartnerPolicyArea,
        PartnerAuditAssessment,
        PartnerReporting,
        PartnerMandateMission,
        PartnerExperience,
        PartnerInternalControl,
        PartnerBudget,
        PartnerFunding,
        PartnerCollaborationPartnership,
        PartnerCollaborationEvidence,
        PartnerOtherInfo,
        PartnerMember,
        PartnerReview,

        EOI,
        Pin,
        Application,
        ApplicationFeedback,
        Assessment,

        PartnerFlag,
        PartnerVerification,

        SanctionedName,
        SanctionedNameMatch,
    ]

    if not settings.IS_PROD:
        print("Start DB Wipe")

        for model in models_to_wipe:
            print(f'Deleting {model.objects.all().count()} {model} instances')
            model.objects.all().delete()

        print("End DB wipe")


USERNAME_AGENCY_ROLE_POSTFIXES = {
    AgencyRole.ADMINISTRATOR.name: 'admin',
    AgencyRole.HQ_EDITOR.name: 'editor-hq',
    AgencyRole.READER.name: 'reader',
    AgencyRole.EDITOR_BASIC.name: 'editor',
    AgencyRole.EDITOR_ADVANCED.name: 'editor-adv',
    AgencyRole.PAM_USER.name: 'pam',
    AgencyRole.MFT_USER.name: 'mft',
}


def generate_fake_data(country_count=3):
    admin, created = User.objects.get_or_create(fullname='admin', defaults={
        'email': '<EMAIL>',
        'is_superuser': True,
        'is_staff': True,
    })
    password = '<PASSWORD>!'
    admin.set_password(password)
    admin.save()
    print("Superuser {}: {}/{}".format("created" if created else "updated", admin.email, password))

    agencies = [
        Agency.objects.get(name="UNICEF"),
        Agency.objects.get(name="WFP"),
        Agency.objects.get(name="UNHCR"),
    ]

    chosen_countries = random.choices(countries, k=country_count)

    OtherAgencyFactory()

    partner_count = 2
    ingo_hqs = [
        PartnerFactory(display_type=PARTNER_TYPES.international) for _ in range(partner_count)
    ]
    for index, hq in enumerate(ingo_hqs):
        PartnerVerificationFactory(partner=hq)
        for role_code, display_name in PartnerRole.get_choices():
            user = UserFactory(
                email=f'ingo-hq-{index + 1}-{role_code.lower()}<EMAIL>'
            )
            PartnerMemberFactory(user=user, role=role_code, partner=hq)
            print(f'Created {user}')

    standard_partners_created = 0
    ingo_partners_created = 0
    for country_code, country_name in chosen_countries:
        for index in range(partner_count):
            for partner_type, display_type in PARTNER_TYPES:
                if partner_type == PARTNER_TYPES.international:
                    name_parts = ingo_hqs[index].legal_name.split(' ') + [country_code]
                    partner_kwargs = {
                        'legal_name': f'{"-".join(name_parts)}.{partner_type}'.lower(),
                        'hq': ingo_hqs[index]
                    }
                    ingo_partners_created += 1
                else:
                    def get_legal_name():
                        name_parts = get_partner_name().split(" ")
                        return f'{"-".join(name_parts)}.{partner_type}'.lower()

                    # Soft unique check
                    legal_name = get_legal_name()
                    while Partner.objects.filter(legal_name=legal_name).exists():
                        legal_name = get_legal_name()

                    partner_kwargs = {
                        'hq': None,
                        'legal_name': legal_name,
                    }
{'fraction_non_alphanumeric': 0.08499558693733451, 'fraction_numerical': 0.001323918799646955, 'mean_word_length': 2.3238486359636257, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4620720', 'n_tokens_mistral': 3328, 'n_tokens_neox': 3344, 'n_words': 585}