<commit_before>from django.core.management.base import BaseCommand
from django.db import connection
from mailchimp_pipeline.client import ChimpClient
from mailchimp_pipeline.helpers import get_user_active_enrollements, get_enrollements_course_short_ids
from django.contrib.auth.models import User
from lms.djangoapps.onboarding.models import FocusArea, OrgSector
from lms.djangoapps.certificates import api as certificate_api
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
from django.conf import settings

from logging import getLogger
log = getLogger(__name__)


class Command(BaseCommand):
    help = """
    One time addition of already existing users into mailchimp learner's list
    example:
        manage.py sync_users_with_mailchimp
    """

    def send_user_to_mailchimp(self, client, users):
        client.add_list_members_in_batch(settings.MAILCHIMP_LEARNERS_LIST_ID, {"members": users})

    def get_users_data_to_send(self, users):
        users_set = []

        focus_areas = FocusArea.get_map()
        org_sectors = OrgSector.get_map()

        for user in users:
            profile = user.profile
            extended_profile = user.extended_profile

            org_type = ""
            if extended_profile.organization and extended_profile.organization.org_type:
                org_type = org_sectors.get(extended_profile.organization.org_type, '')

            all_certs = []
            try:
                all_certs = certificate_api.get_certificates_for_user(user.username)
            except Exception as ex:
                log.exception(str(ex.args))

            completed_course_keys = [cert.get('course_key', '') for cert in all_certs
                                     if certificate_api.is_passing_status(cert['status'])]
            completed_courses = CourseOverview.objects.filter(id__in=completed_course_keys)

            try:
                user_json = {
                    "email_address": user.email,
                    "status_if_new": "subscribed",
                    "merge_fields": {
                        "FULLNAME": user.get_full_name(),
                        "USERNAME": user.username,
                        "LANG": profile.language if profile.language else "",
                        "COUNTRY": profile.country.name.format() if profile.country else "",
                        "CITY": profile.city if profile.city else "",
                        "DATEREGIS": str(user.date_joined.strftime("%m/%d/%Y")),
                        "LSOURCE": "",
                        "COMPLETES": ", ".join([course.display_name for course in completed_courses]),
                        "ENROLLS": get_user_active_enrollements(user.username),
                        "ENROLL_IDS": get_enrollements_course_short_ids(user.username),
                        "ORG": extended_profile.organization.label if extended_profile.organization else "",
                        "ORGTYPE": org_type,
                        "WORKAREA": str(focus_areas.get(extended_profile.organization.focus_area, ""))
                        if extended_profile.organization else "",
                    }
                }
            except Exception as ex:
                log.info("There was an error for user with email address as {}".format(user.email))
                log.exception(str(ex.args))
                continue

            users_set.append(user_json)

        return users_set

    def handle(self, *args, **options):
        batch_size = 500
        cursor = connection.cursor()
        cursor.execute('SET TRANSACTION ISOLATION LEVEL READ COMMITTED')
        client = ChimpClient()
        total_user_count = User.objects.all().count()<commit_msg>Handle RelatedObjectDoesNotExist exception<commit_after>from django.core.management.base import BaseCommand
from django.db import connection
from mailchimp_pipeline.client import ChimpClient
from mailchimp_pipeline.helpers import get_user_active_enrollements, get_enrollements_course_short_ids
from django.contrib.auth.models import User
from lms.djangoapps.onboarding.models import FocusArea, OrgSector
from lms.djangoapps.certificates import api as certificate_api
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
from django.conf import settings

from logging import getLogger
log = getLogger(__name__)


class Command(BaseCommand):
    help = """
    One time addition of already existing users into mailchimp learner's list
    example:
        manage.py sync_users_with_mailchimp
    """

    def send_user_to_mailchimp(self, client, users):
        client.add_list_members_in_batch(settings.MAILCHIMP_LEARNERS_LIST_ID, {
            "members": users,
            "update_existing": True
        })

    def get_users_data_to_send(self, users):
        users_set = []

        focus_areas = FocusArea.get_map()
        org_sectors = OrgSector.get_map()

        for user in users:
            language = country = city = organization = org_type = work_area = ""
            profile = extended_profile = None
            try:
                profile = user.profile
                extended_profile = user.extended_profile

                if profile.language:
                    language = profile.language

                if profile.country:
                    country = profile.country.name.format()

                if profile.city:
                    city = profile.city

                if extended_profile.organization:
                    organization = extended_profile.organization.label
                    work_area = str(focus_areas.get(
                        extended_profile.organization.focus_area, ""
                    ))
                    if extended_profile.organization.org_type:
                        org_type = org_sectors.get(
                            extended_profile.organization.org_type, ''
                        )
            except:
                pass

            all_certs = []
            try:
                all_certs = certificate_api.get_certificates_for_user(user.username)
            except Exception as ex:
                log.exception(str(ex.args))

            completed_course_keys = [cert.get('course_key', '') for cert in all_certs
                                     if certificate_api.is_passing_status(cert['status'])]
            completed_courses = CourseOverview.objects.filter(id__in=completed_course_keys)

            try:
                user_json = {
                    "email_address": user.email,
                    "status_if_new": "subscribed",
                    "merge_fields": {
                        "FULLNAME": user.get_full_name(),
                        "USERNAME": user.username,
                        "LANG": language,
                        "COUNTRY": country,
                        "CITY": city,
                        "DATEREGIS": str(user.date_joined.strftime("%m/%d/%Y")),
                        "LSOURCE": "",
                        "COMPLETES": ", ".join([course.display_name for course in completed_courses]),
                        "ENROLLS": get_user_active_enrollements(user.username),
                        "ENROLL_IDS": get_enrollements_course_short_ids(user.username),
                        "ORG": organization,
                        "ORGTYPE": org_type,
                        "WORKAREA": work_area,
                    }
                }
            except Exception as ex:
                log.info("There was an error for user with email address as {}".format(user.email))
                log.exception(str(ex.args))
                continue

            users_set.append(user_json)

        return users_set

    def handle(self, *args, **options):
        batch_size = 500
        cursor = connection.cursor()
        cursor.execute('SET TRANSACTION ISOLATION LEVEL READ COMMITTED')
        client = ChimpClient()
        total_user_count = User.objects.all().count()
{'fraction_non_alphanumeric': 0.08212985055883461, 'fraction_numerical': 0.0007534848675122441, 'mean_word_length': 1.916147931160747, 'pattern_counts': {'":': 35, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6081447', 'n_tokens_mistral': 2132, 'n_tokens_neox': 1993, 'n_words': 419}