<commit_before>import csv
from decimal import Decimal
import math
import logging
import traceback
import sys
from collections import namedtuple
from optparse import make_option
from bluebottle.bb_projects.models import ProjectPhase

import os
from django.core.management.base import BaseCommand
from django.utils import timezone
from apps.cowry_docdata.adapters import WebDirectDocDataDirectDebitPaymentAdapter
from apps.cowry_docdata.exceptions import DocDataPaymentException
from apps.cowry_docdata.models import DocDataPaymentOrder
from apps.fund.models import RecurringDirectDebitPayment, Order, OrderStatuses, Donation
from apps.projects.models import Project
from ...mails import mail_monthly_donation_processed_notification

logger = logging.getLogger(__name__)


#
# Run with:
# ./manage.py process_monthly_donations -v 2 --settings=bluebottle.settings.local (or .production etc.)
#

class Command(BaseCommand):
    help = 'Process monthly donations.'
    requires_model_validation = True

    verbosity_loglevel = {
        '0': logging.ERROR,    # 0 means no output.
        '1': logging.WARNING,  # 1 means normal output (default).
        '2': logging.INFO,     # 2 means verbose output.
        '3': logging.DEBUG     # 3 means very verbose output.
    }

    option_list = BaseCommand.option_list + (
        make_option('--dry-run', action='store_true', dest='dry_run', default=False,
                    help="Process the monthly donations without creating any db records or payments."),

        make_option('--no-email', action='store_true', dest='no_email', default=False,
                    help="Don't send the monthly donation email to users."),

        make_option('--csv-export', action='store_true', dest='csv_export', default=False,
                    help="Generate CSV export of monthly donors with donations amounts."),

        make_option('--process-one-recurring-payment', action='store', dest='process_payment_id', type='int',
                    metavar='RECURRING-PAYMENT-ID',
                    help="Process only the RecurringDirectDebitPayment specified by its primary key."),
    )

    def handle(self, *args, **options):
        # Setup the log level for root logger.
        loglevel = self.verbosity_loglevel.get(options['verbosity'])
        logger.setLevel(loglevel)

        if options['dry_run'] and options['csv_export']:
            logger.error("You cannot set both '--dry-run' and '--csv-export'.")
            sys.exit(1)

        send_email = not options['no_email']

        if options['dry_run']:
            # TODO Implement --dry-run.
            logger.warn("Config: --dry-run not fully implemented yet. Database records and payments will be created.")
            logger.info("Config: Not sending emails.")
            send_email = False

        recurring_payments_queryset = RecurringDirectDebitPayment.objects.filter(active=True, manually_process=False)
        if options['csv_export']:
            generate_monthly_donations_csv(recurring_payments_queryset)
        else:
            if options['process_payment_id']:
                recurring_payments_queryset = recurring_payments_queryset.filter(id=options['process_payment_id'])
            try:
                process_monthly_donations(recurring_payments_queryset, send_email)
            except:
                print traceback.format_exc()


def generate_monthly_donations_csv(recurring_payments_queryset):
    csv_path = os.path.expanduser('~/monthly-donors-{0}.csv'.format(timezone.now().date()))
    logger.info("Saving monthly donations CSV file to:")
    logger.info("  {0}".format(csv_path))
    with open(csv_path, 'wb') as csv_file:
        csvwriter = csv.writer(csv_file, dialect='excel')
        csvwriter.writerow(['Member', 'Active', 'Amount'])
        for rp in recurring_payments_queryset:
            csvwriter.writerow([rp.user.email, rp.active, rp.amount])


def update_last_donation(donation, remaining_amount, popular_projects):
    """
    Updates the last donation with the remaining amount of the payment. If the donation is more than the project
    needs, the project will be filled and the balance will be used to fill the popular projects recursively.
    """
    project = Project.objects.get(id=donation.project_id)

    # Base case.
    if project.amount_donated + remaining_amount <= project.amount_asked or \
            len(popular_projects) == 0:
        # The remaining amount won't fill up the project or we have no more projects to try. We're done.
        logger.debug(u"Donation is less than project '{0}' needs. No further adjustments are needed.".format(project.title))
        donation.amount = remaining_amount
        donation.donation_type = Donation.DonationTypes.recurring
        donation.save()
        return

    # Recursive case.
    else:
        # Fill up the project.
        logger.debug(u"Donation is more than project '{0}' needs. Filling up project and creating new donation.".format(project.title))
        donation.amount = project.amount_asked - project.amount_donated
        donation.donation_type = Donation.DonationTypes.recurring
        donation.save()

        # Create a new Donation and recursively update it with the remaining amount.
        new_project = popular_projects.pop(0)
        new_donation = Donation.objects.create(user=donation.user, project=new_project, amount=0, currency='EUR',
                                               donation_type=Donation.DonationTypes.recurring, order=donation.order)
        update_last_donation(new_donation, remaining_amount - donation.amount, popular_projects)


def create_recurring_order(user, projects, order=None):
    """
    Creates a recurring Order with donations to the supplied projects.
    """
    if not order:
        order = Order.objects.create(status=OrderStatuses.recurring, user=user, recurring=True)

    for p in projects:
        project = Project.objects.get(id=p.id)
        if project.status == ProjectPhase.objects.get(slug="campaign"):
            Donation.objects.create(user=user, project=project, amount=0, currency='EUR',
                                    donation_type=Donation.DonationTypes.recurring, order=order)
    return order


def remove_order(order):
    for donation in order.donations.all():
        donation.delete()
    order.delete()


def correct_donation_amounts(popular_projects, recurring_order, recurring_payment):
    """
    Divides the total amount for the monthly donation across all projects. This method deals with the case of a
    donation filling up a project.
    """
    remaining_amount = recurring_payment.amount
    num_donations = recurring_order.donations.count()
    amount_per_project = Decimal(math.floor(recurring_payment.amount / num_donations))
    donations = recurring_order.donations.all()
    logger.info("Really! {0} donations".format(len(donations)))
    for i in range(0, num_donations - 1):
        donation = donations[i]
        project = Project.objects.get(id=donation.project_id)
        if project.amount_donated + amount_per_project > project.amount_asked:
            donation.amount = project.amount_asked - project.amount_donated
        else:
            donation.amount = amount_per_project
        donation.donation_type = Donation.DonationTypes.recurring
        donation.save()
        remaining_amount -= donation.amount
    update_last_donation(donations[num_donations - 1], remaining_amount, popular_projects)


def process_monthly_donations(recurring_payments_queryset, send_email):
    """ The starting point for creating DocData payments for the monthly donations. """

    recurring_donation_errors = []
    RecurringDonationError = namedtuple('RecurringDonationError', 'recurring_payment error_message')
    skipped_recurring_payments = []
    SkippedRecurringPayment = namedtuple('SkippedRecurringPayment', 'recurring_payment orders')<commit_msg>Fix monthly donations for projec amounts in euros.<commit_after>import csv
from decimal import Decimal
import math
import logging
import traceback
import sys
from collections import namedtuple
from optparse import make_option
from bluebottle.bb_projects.models import ProjectPhase

import os
from django.core.management.base import BaseCommand
from django.utils import timezone
from apps.cowry_docdata.adapters import WebDirectDocDataDirectDebitPaymentAdapter
from apps.cowry_docdata.exceptions import DocDataPaymentException
from apps.cowry_docdata.models import DocDataPaymentOrder
from apps.fund.models import RecurringDirectDebitPayment, Order, OrderStatuses, Donation
from apps.projects.models import Project
from ...mails import mail_monthly_donation_processed_notification

logger = logging.getLogger(__name__)


#
# Run with:
# ./manage.py process_monthly_donations -v 2 --settings=bluebottle.settings.local (or .production etc.)
#

class Command(BaseCommand):
    help = 'Process monthly donations.'
    requires_model_validation = True

    verbosity_loglevel = {
        '0': logging.ERROR,    # 0 means no output.
        '1': logging.WARNING,  # 1 means normal output (default).
        '2': logging.INFO,     # 2 means verbose output.
        '3': logging.DEBUG     # 3 means very verbose output.
    }

    option_list = BaseCommand.option_list + (
        make_option('--dry-run', action='store_true', dest='dry_run', default=False,
                    help="Process the monthly donations without creating any db records or payments."),

        make_option('--no-email', action='store_true', dest='no_email', default=False,
                    help="Don't send the monthly donation email to users."),

        make_option('--csv-export', action='store_true', dest='csv_export', default=False,
                    help="Generate CSV export of monthly donors with donations amounts."),

        make_option('--process-one-recurring-payment', action='store', dest='process_payment_id', type='int',
                    metavar='RECURRING-PAYMENT-ID',
                    help="Process only the RecurringDirectDebitPayment specified by its primary key."),
    )

    def handle(self, *args, **options):
        # Setup the log level for root logger.
        loglevel = self.verbosity_loglevel.get(options['verbosity'])
        logger.setLevel(loglevel)

        if options['dry_run'] and options['csv_export']:
            logger.error("You cannot set both '--dry-run' and '--csv-export'.")
            sys.exit(1)

        send_email = not options['no_email']

        if options['dry_run']:
            # TODO Implement --dry-run.
            logger.warn("Config: --dry-run not fully implemented yet. Database records and payments will be created.")
            logger.info("Config: Not sending emails.")
            send_email = False

        recurring_payments_queryset = RecurringDirectDebitPayment.objects.filter(active=True, manually_process=False)
        if options['csv_export']:
            generate_monthly_donations_csv(recurring_payments_queryset)
        else:
            if options['process_payment_id']:
                recurring_payments_queryset = recurring_payments_queryset.filter(id=options['process_payment_id'])
            try:
                process_monthly_donations(recurring_payments_queryset, send_email)
            except:
                print traceback.format_exc()


def generate_monthly_donations_csv(recurring_payments_queryset):
    csv_path = os.path.expanduser('~/monthly-donors-{0}.csv'.format(timezone.now().date()))
    logger.info("Saving monthly donations CSV file to:")
    logger.info("  {0}".format(csv_path))
    with open(csv_path, 'wb') as csv_file:
        csvwriter = csv.writer(csv_file, dialect='excel')
        csvwriter.writerow(['Member', 'Active', 'Amount'])
        for rp in recurring_payments_queryset:
            csvwriter.writerow([rp.user.email, rp.active, rp.amount])


def update_last_donation(donation, remaining_amount, popular_projects):
    """
    Updates the last donation with the remaining amount of the payment. If the donation is more than the project
    needs, the project will be filled and the balance will be used to fill the popular projects recursively.
    """
    project = Project.objects.get(id=donation.project_id)

    # Base case.
    if project.amount_donated + (remaining_amount/100) <= project.amount_asked or len(popular_projects) == 0:
        # The remaining amount won't fill up the project or we have no more projects to try. We're done.
        logger.debug(u"Donation is less than project '{0}' needs. No further adjustments are needed.".format(project.title))
        donation.amount = remaining_amount
        donation.donation_type = Donation.DonationTypes.recurring
        donation.save()
        return

    # Recursive case.
    else:
        # Fill up the project.
        logger.debug(u"Donation is more than project '{0}' needs. Filling up project and creating new donation.".format(project.title))
        donation.amount = (project.amount_asked - project.amount_donated) * 100
        donation.donation_type = Donation.DonationTypes.recurring
        donation.save()

        # Create a new Donation and recursively update it with the remaining amount.
        new_project = popular_projects.pop(0)
        new_donation = Donation.objects.create(user=donation.user, project=new_project, amount=0, currency='EUR',
                                               donation_type=Donation.DonationTypes.recurring, order=donation.order)
        update_last_donation(new_donation, remaining_amount - donation.amount, popular_projects)


def create_recurring_order(user, projects, order=None):
    """
    Creates a recurring Order with donations to the supplied projects.
    """
    if not order:
        order = Order.objects.create(status=OrderStatuses.recurring, user=user, recurring=True)

    for p in projects:
        project = Project.objects.get(id=p.id)
        if project.status == ProjectPhase.objects.get(slug="campaign"):
            Donation.objects.create(user=user, project=project, amount=0, currency='EUR',
                                    donation_type=Donation.DonationTypes.recurring, order=order)
    return order


def remove_order(order):
    for donation in order.donations.all():
        donation.delete()
    order.delete()


def correct_donation_amounts(popular_projects, recurring_order, recurring_payment):
    """
    Divides the total amount for the monthly donation across all projects. This method deals with the case of a
    donation filling up a project.
    """
    remaining_amount = recurring_payment.amount
    num_donations = recurring_order.donations.count()
    amount_per_project = Decimal(math.floor(recurring_payment.amount / num_donations))
    donations = recurring_order.donations.all()
    logger.info("Processing {0} donations".format(len(donations)))
    for i in range(0, num_donations - 1):
        donation = donations[i]
        project = Project.objects.get(id=donation.project_id)
        if project.amount_donated + (amount_per_project/100) > project.amount_asked:
            donation.amount = 100 * (project.amount_asked - project.amount_donated)
        else:
            donation.amount = amount_per_project
        donation.donation_type = Donation.DonationTypes.recurring
        donation.save()
        remaining_amount -= donation.amount
    update_last_donation(donations[num_donations - 1], remaining_amount, popular_projects)


def process_monthly_donations(recurring_payments_queryset, send_email):
    """ The starting point for creating DocData payments for the monthly donations. """

    recurring_donation_errors = []
    RecurringDonationError = namedtuple('RecurringDonationError', 'recurring_payment error_message')
    skipped_recurring_payments = []
    SkippedRecurringPayment = namedtuple('SkippedRecurringPayment', 'recurring_payment orders')
{'fraction_non_alphanumeric': 0.09254352763058289, 'fraction_numerical': 0.0035326772646984608, 'mean_word_length': 3.3420980553273076, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5395114', 'n_tokens_mistral': 4550, 'n_tokens_neox': 4360, 'n_words': 1100}