<commit_before>import hashlib
import os
import sys

from django.conf import settings
from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist

import commonware.log
import cronjobs
from celery.task.sets import TaskSet
from celeryutils import chunked

import larper
from users.models import UserProfile

log = commonware.log.getLogger('m.cron')


@cronjobs.register
def create_missing_profiles():
    """Create missing profiles on dev/stage.

    The dev/staging servers are missing certain User objects, likely because
    of users created before our UserProfile signal receivers. This will
    create them and prevent us from getting annoying tracebacks from tests.
    """
    users = User.objects.all()

    for u in users:
        try:
            u.get_profile()
        except ObjectDoesNotExist:
            log.info('Created profile for user with email %s' % u.email)
            UserProfile.objects.create(user=u)


@cronjobs.register
def create_missing_users():
    """Create missing users on dev/stage.

    The dev/staging servers are missing certain User objects, likely because
    of users created before our UserProfile signal receivers. This will
    create them and prevent us from getting annoying tracebacks from tests.

    Note that this needs to be run with the LDAP limit (temporarily) disabled.
    """
    users = larper._return_all()
    users_created = []

    for u in users:
        email = u[1]['mail'][0]
        if not User.objects.filter(email=email):
            try:
                User.objects.create(username=email, email=email)
                users_created.append(email)
                log.info('Created user for LDAPer with email %s' % email)
            except:
                log.info('Could not create user for LDAPer with email %s'
                         % email)

    log.info('Created %s users' % len(users_created))


@cronjobs.register
def vouchify():
    """Synchronizes LDAP vouch info into database.

    This queries LDAP for users who's corresponding ``UserProfile`` has
    ``is_vouched`` as ``False``.  It then updates ``is_vouched`` and
    ``vouched_by`` with up-to-date data.
    """
    users = UserProfile.objects.filter(is_vouched=False)

    for user in users:
        person = user.get_ldap_person()
        if person and 'mozilliansVouchedBy' in person[1]:
            user.is_vouched = True
            voucher = (person[1]['mozilliansVouchedBy'][0].split(',')[0]
                                                          .split('=')[1])
            by = larper.get_user_by_uid(voucher)
            if by:
                email = by[1]['mail'][0]
                try:
                    user.vouched_by = (User.objects.get(email=email)
                                                   .get_profile())
                except User.DoesNotExist:
                    log.warning('No matching user for %s' % email)
                except UserProfile.DoesNotExist:
                    log.warning('No matching user_profile for %s' % email)
            user.save()
            log.info('Data copied for %s' % user.user.username)
        log.debug('%s is still unvouched... skipping' % user.user.username)


# TODO: Remove after we can safely say that LDAP is gone.
@cronjobs.register
def fix_bad_ldap_vouch():
    """Synchronizes MySQL vouch data into broken LDAP accounts.

    Just vouches them as ZUUL because the MySQL vouch data is good and what
    we'll use going forward.
    """
    users = UserProfile.objects.filter(is_vouched=True)

    for user in users:
        person = user.get_ldap_person()

        if person:
            try:
                larper.record_vouch('ZUUL', person[1]['uniqueIdentifier'][0])
            except TYPE_OR_VALUE_EXISTS:
                log.info('User is already vouched; no big.')


def _populate_from_ldap(profile):
    person = profile.get_ldap_person()
    if not person:
        log.warning('No ldap data for %d' % profile.user_id)
        return
    username = person[1]['uniqueIdentifier'][0]
    bio = firstname = ircname = ''
    photo = False

    if 'description' in person[1]:
        bio = person[1]['description'][0]
    if 'jpegPhoto' in person[1]:
        photo = person[1]['jpegPhoto'][0]
    lastname = person[1]['sn'][0]
    firstname = ''
    if 'givenName' in person[1]:
        firstname = person[1]['givenName'][0]
    displayname = person[1]['displayName'][0]
    service_data = larper.get_service_data(username)

    if service_data:
        ircname = service_data['irc://irc.mozilla.org/'].service_id

    if photo:
        # ensure the netapp storage exists
        if not os.path.exists(settings.NETAPP_STORAGE):
            log.error('Netapp storage not found at %s'
                       % settings.NETAPP_STORAGE)
            sys.exit(1)

        # if userpic dir doesn't exist make it
        if not os.path.exists(settings.USERPICS_PATH):
            os.mkdir(settings.USERPICS_PATH)

        destination = os.path.join(settings.USERPICS_PATH, '%d.jpg' %
                                   profile.id)

        md5 = lambda x: hashlib.md5(x).hexdigest()

        write_it = True
        # see if file already exists for user
        if os.path.exists(destination):
            # if it does compare it
            ldap_hash = md5(photo)
            existing_hash = md5(open(destination).read())
            if ldap_hash == existing_hash:
                # if same log debug
                log.debug('File already dumped for user: %d' %
                          profile.id)
                write_it = False
            else:
                # if different log warning and copy it
                log.warning('Different file in place for user: %d' %
                            profile.id)
                os.rename(destination, destination + '.' + existing_hash)

        if write_it:
            with open(destination, 'w') as f:
                f.write(photo)

    # append underscore's until we find something unique
    while True:
        if not User.objects.filter(username=username).count():
            break
        log.warning('duplicat username found: %s' % username)
        username = username + '_'

    user = profile.user

    user.username = username
    user.lastname = lastname
    user.firstname = firstname
    user.save()

    profile.bio = bio
    profile.photo = bool(photo)
    profile.display_name = displayname
    profile.ircname = ircname
    profile.save()
    log.debug('u:%d saved' % profile.user_id)


@cronjobs.register
def flee_ldap_noname():
    """Copies data from LDAP into UserProfile."""
    users = User.objects.filter(last_name='')
    print '%d profiles' % len(users)
    for user in users:
        print 'User: %d, %s' % (user.id, user.email)
        _populate_from_ldap(user.get_profile())


@cronjobs.register
def flee_ldap_nodisplay():
    """Copies data from LDAP into UserProfile."""
    profiles = UserProfile.objects.filter(display_name='')
    log.info('%d profiles' % len(profiles))<commit_msg>Fix some typos<commit_after>import hashlib
import os
import sys

from django.conf import settings
from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist

import commonware.log
import cronjobs
from celery.task.sets import TaskSet
from celeryutils import chunked

import larper
from users.models import UserProfile

log = commonware.log.getLogger('m.cron')


@cronjobs.register
def create_missing_profiles():
    """Create missing profiles on dev/stage.

    The dev/staging servers are missing certain User objects, likely because
    of users created before our UserProfile signal receivers. This will
    create them and prevent us from getting annoying tracebacks from tests.
    """
    users = User.objects.all()

    for u in users:
        try:
            u.get_profile()
        except ObjectDoesNotExist:
            log.info('Created profile for user with email %s' % u.email)
            UserProfile.objects.create(user=u)


@cronjobs.register
def create_missing_users():
    """Create missing users on dev/stage.

    The dev/staging servers are missing certain User objects, likely because
    of users created before our UserProfile signal receivers. This will
    create them and prevent us from getting annoying tracebacks from tests.

    Note that this needs to be run with the LDAP limit (temporarily) disabled.
    """
    users = larper._return_all()
    users_created = []

    for u in users:
        email = u[1]['mail'][0]
        if not User.objects.filter(email=email):
            try:
                User.objects.create(username=email, email=email)
                users_created.append(email)
                log.info('Created user for LDAPer with email %s' % email)
            except:
                log.info('Could not create user for LDAPer with email %s'
                         % email)

    log.info('Created %s users' % len(users_created))


@cronjobs.register
def vouchify():
    """Synchronizes LDAP vouch info into database.

    This queries LDAP for users who's corresponding ``UserProfile`` has
    ``is_vouched`` as ``False``.  It then updates ``is_vouched`` and
    ``vouched_by`` with up-to-date data.
    """
    users = UserProfile.objects.filter(is_vouched=False)

    for user in users:
        person = user.get_ldap_person()
        if person and 'mozilliansVouchedBy' in person[1]:
            user.is_vouched = True
            voucher = (person[1]['mozilliansVouchedBy'][0].split(',')[0]
                                                          .split('=')[1])
            by = larper.get_user_by_uid(voucher)
            if by:
                email = by[1]['mail'][0]
                try:
                    user.vouched_by = (User.objects.get(email=email)
                                                   .get_profile())
                except User.DoesNotExist:
                    log.warning('No matching user for %s' % email)
                except UserProfile.DoesNotExist:
                    log.warning('No matching user_profile for %s' % email)
            user.save()
            log.info('Data copied for %s' % user.user.username)
        log.debug('%s is still unvouched... skipping' % user.user.username)


# TODO: Remove after we can safely say that LDAP is gone.
@cronjobs.register
def fix_bad_ldap_vouch():
    """Synchronizes MySQL vouch data into broken LDAP accounts.

    Just vouches them as ZUUL because the MySQL vouch data is good and what
    we'll use going forward.
    """
    users = UserProfile.objects.filter(is_vouched=True)

    for user in users:
        person = user.get_ldap_person()

        if person:
            try:
                larper.record_vouch('ZUUL', person[1]['uniqueIdentifier'][0])
            except TYPE_OR_VALUE_EXISTS:
                log.info('User is already vouched; no big.')


def _populate_from_ldap(profile):
    person = profile.get_ldap_person()
    if not person:
        log.warning('No ldap data for %d' % profile.user_id)
        return
    username = person[1]['uniqueIdentifier'][0]
    bio = firstname = ircname = ''
    photo = False

    if 'description' in person[1]:
        bio = person[1]['description'][0]
    if 'jpegPhoto' in person[1]:
        photo = person[1]['jpegPhoto'][0]
    lastname = person[1]['sn'][0]
    firstname = ''
    if 'givenName' in person[1]:
        firstname = person[1]['givenName'][0]
    displayname = person[1]['displayName'][0]
    service_data = larper.get_service_data(username)

    if service_data:
        ircname = service_data['irc://irc.mozilla.org/'].service_id

    if photo:
        # ensure the netapp storage exists
        if not os.path.exists(settings.NETAPP_STORAGE):
            log.error('Netapp storage not found at %s'
                       % settings.NETAPP_STORAGE)
            sys.exit(1)

        # if userpic dir doesn't exist make it
        if not os.path.exists(settings.USERPICS_PATH):
            os.mkdir(settings.USERPICS_PATH)

        destination = os.path.join(settings.USERPICS_PATH, '%d.jpg' %
                                   profile.id)

        md5 = lambda x: hashlib.md5(x).hexdigest()

        write_it = True
        # see if file already exists for user
        if os.path.exists(destination):
            # if it does compare it
            ldap_hash = md5(photo)
            existing_hash = md5(open(destination).read())
            if ldap_hash == existing_hash:
                # if same log debug
                log.debug('File already dumped for user: %d' %
                          profile.id)
                write_it = False
            else:
                # if different log warning and copy it
                log.warning('Different file in place for user: %d' %
                            profile.id)
                os.rename(destination, destination + '.' + existing_hash)

        if write_it:
            with open(destination, 'w') as f:
                f.write(photo)

    # append underscore's until we find something unique
    while True:
        if not User.objects.filter(username=username).count():
            break
        log.warning('duplicat username found: %s' % username)
        username = username + '_'

    user = profile.user

    user.username = username
    user.last_name = lastname
    user.first_name = firstname
    user.save()

    profile.bio = bio
    profile.photo = bool(photo)
    profile.display_name = displayname
    profile.ircname = ircname
    profile.save()
    log.debug('u:%d saved' % profile.user_id)


@cronjobs.register
def flee_ldap_noname():
    """Copies data from LDAP into UserProfile."""
    users = User.objects.filter(last_name='')
    print '%d profiles' % len(users)
    for user in users:
        print 'User: %d, %s' % (user.id, user.email)
        _populate_from_ldap(user.get_profile())


@cronjobs.register
def flee_ldap_nodisplay():
    """Copies data from LDAP into UserProfile."""
    profiles = UserProfile.objects.filter(display_name='')
    log.info('%d profiles' % len(profiles))
{'fraction_non_alphanumeric': 0.09938153124333547, 'fraction_numerical': 0.004407478495770243, 'mean_word_length': 2.209673739447867, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7346477', 'n_tokens_mistral': 4140, 'n_tokens_neox': 3953, 'n_words': 1133}