<commit_before># encoding: UTF-8
from __future__ import with_statement

import os
import signal
import logging

from threading import currentThread
from subprocess import Popen

from datetime import datetime, timedelta
from decimal import Decimal
from string import Template
from optparse import make_option

from django.db.models import Count
from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ObjectDoesNotExist
from django.utils.translation import ugettext as _
from django.conf import settings

from membership.models import BillingCycle
from membership.utils import log_change
from membership.reference_numbers import barcode_4

logger = logging.getLogger("paper_bills")
TMPDIR = '/tmp/sikteeritex'

class LatexTemplate(Template):
    delimiter = '\$'

class Timeout(Exception):
    pass

def timeout_handler(signum, frame):
    raise Timeout

def get_data(memberid=None):
    # TODO: refactor central parts into classmethod BillingCycle.paper_reminders
    # TODO: rename (data is never a good name for anything)
    if not settings.ENABLE_REMINDERS:
        return BillingCycle.objects.none()

    qs = BillingCycle.objects

    # Single membership case
    if memberid:
        logger.info('memberid: %s' % memberid)
        qs = qs.filter(membership__id=memberid)
        qs = qs.exclude(bill__type='P')
        return qs

    # For all memberships in Approved state
    qs = qs.annotate(bills=Count('bill'))
    qs = qs.filter(bills__gt=2,
                   is_paid__exact=False,
                   membership__status='A',
                   membership__id__gt=-1)
    qs = qs.exclude(bill__type='P')
    qs = qs.order_by('start')

    return qs

def tex_sanitize(input_string):
    return input_string.replace("#","\#").replace("$", "\$")

def prettyname(cycle):
    # TODO: use translations
    return u"%d & Jäsenmaksu kaudelle & %s - %s \\\\\n" % (1, 
                    cycle.start.strftime('%d.%m.%Y'), cycle.end.strftime('%d.%m.%Y'))

def data2pdf(data):
    # TODO: use Django templates, use with-statement
    t = LatexTemplate(open(settings.PAPER_REMINDER_TEMPLATE).read().decode("UTF-8"))
    targetfile = "m_%d.tex" % int(data['JASENNRO'])
    targetfile = os.path.join(TMPDIR, targetfile)
    target = open(targetfile, 'w')
    target.write(t.safe_substitute(data).encode("UTF-8"))
    target.close()

    try:
        return generate_pdf(targetfile)
    except RuntimeError as e:
        logger.exception(e)
        print("Failed to generate pdf for member %d" % int(data['JASENNRO']))
        print("Data was %s" % data)
    return None

def create_datalist(memberid=None):
    # TODO: template variable names in English
    # TODO: use Django SHORT_DATE_FORMAT
    # TODO: do LaTeX-specific escaping utilities belong into an application-wide latex_utils.py?
    datalist = []
    for cycle in get_data(memberid).all():
        # check if paper reminder already sent
        cont = False
        for bill in cycle.bill_set.all():
            if bill.type == 'P':
                cont=True
                break
        if cont:
            continue

        membercontact = cycle.membership.get_billing_contact()
        data = {
            'DATE'      : datetime.now().strftime("%d.%m.%Y"),
            'JASENNRO'  : cycle.membership.id,
            'NIMI'      : tex_sanitize(cycle.membership.name()),
            'SUMMA'     : cycle.sum,
            'VIITENRO'  : cycle.reference_number,
            'MAKSUDATA' : prettyname(cycle),
            'EMAIL'     : tex_sanitize(membercontact.email.replace("_", "\_")),
            'OSOITE'    : tex_sanitize(membercontact.street_address),
            'POSTI'     : tex_sanitize("%s %s" % (membercontact.postal_code, membercontact.post_office)),
            'BARCODE'   : barcode_4(settings.IBAN_ACCOUNT_NUMBER,cycle.reference_number,None,cycle.sum)
        }
        datalist.append(data)
    return datalist

def generate_pdf(latexfile):
    # TODO: with_timeouting_process(...)
    # set umask to 0077
    oldumask = os.umask(63)
    pid = Popen(['pdflatex', '-interaction=batchmode', '-output-directory=%s' % TMPDIR, 
                '-no-file-line-error','-halt-on-error','-output-format','pdf',latexfile])
    # wait until process ends
    pid.wait()
    try:
        os.remove(latexfile)
    except OSError as e:
        logging.exception(e)<commit_msg>Added vat to paper reminders<commit_after># encoding: UTF-8
from __future__ import with_statement

import os
import signal
import logging

from threading import currentThread
from subprocess import Popen

from datetime import datetime, timedelta
from decimal import Decimal
from string import Template
from optparse import make_option

from django.db.models import Count
from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ObjectDoesNotExist
from django.utils.translation import ugettext as _
from django.conf import settings

from membership.models import BillingCycle
from membership.utils import log_change
from membership.reference_numbers import barcode_4

logger = logging.getLogger("paper_bills")
TMPDIR = '/tmp/sikteeritex'

class LatexTemplate(Template):
    delimiter = '\$'

class Timeout(Exception):
    pass

def timeout_handler(signum, frame):
    raise Timeout

def get_data(memberid=None):
    # TODO: refactor central parts into classmethod BillingCycle.paper_reminders
    # TODO: rename (data is never a good name for anything)
    if not settings.ENABLE_REMINDERS:
        return BillingCycle.objects.none()

    qs = BillingCycle.objects

    # Single membership case
    if memberid:
        logger.info('memberid: %s' % memberid)
        qs = qs.filter(membership__id=memberid)
        qs = qs.exclude(bill__type='P')
        return qs

    # For all memberships in Approved state
    qs = qs.annotate(bills=Count('bill'))
    qs = qs.filter(bills__gt=2,
                   is_paid__exact=False,
                   membership__status='A',
                   membership__id__gt=-1)
    qs = qs.exclude(bill__type='P')
    qs = qs.order_by('start')

    return qs

def tex_sanitize(input_string):
    return input_string.replace("#","\#").replace("$", "\$")

def prettyname(cycle):
    # TODO: use translations
    return u"%d & Jäsenmaksu kaudelle & %s - %s & %s \\\\\n" % (1,
                    cycle.start.strftime('%d.%m.%Y'),
                    cycle.end.strftime('%d.%m.%Y'), cycle.sum)

def data2pdf(data):
    # TODO: use Django templates, use with-statement
    t = LatexTemplate(open(settings.PAPER_REMINDER_TEMPLATE).read().decode("UTF-8"))
    targetfile = "m_%d.tex" % int(data['JASENNRO'])
    targetfile = os.path.join(TMPDIR, targetfile)
    target = open(targetfile, 'w')
    target.write(t.safe_substitute(data).encode("UTF-8"))
    target.close()

    try:
        return generate_pdf(targetfile)
    except RuntimeError as e:
        logger.exception(e)
        print("Failed to generate pdf for member %d" % int(data['JASENNRO']))
        print("Data was %s" % data)
    return None

def create_datalist(memberid=None):
    # TODO: template variable names in English
    # TODO: use Django SHORT_DATE_FORMAT
    # TODO: do LaTeX-specific escaping utilities belong into an application-wide latex_utils.py?
    datalist = []
    for cycle in get_data(memberid).all():
        # check if paper reminder already sent
        cont = False
        for bill in cycle.bill_set.all():
            if bill.type == 'P':
                cont=True
                break
        if cont:
            continue

        membercontact = cycle.membership.get_billing_contact()
        full = Decimal(100.0)
        vat = (cycle.sum * (cycle.get_vat_percentage() / full))
        amount = cycle.sum - vat
        data = {
            'DATE'      : datetime.now().strftime("%d.%m.%Y"),
            'JASENNRO'  : cycle.membership.id,
            'NIMI'      : tex_sanitize(cycle.membership.name()),
            'AMOUNT'    : "%.2f" % amount,
            'SUM'       : "%.2f" % cycle.sum,
            'VAT'       : "%.2f" % vat,
            'VIITENRO'  : cycle.reference_number,
            'MAKSUDATA' : prettyname(cycle),
            'EMAIL'     : tex_sanitize(membercontact.email.replace("_", "\_")),
            'OSOITE'    : tex_sanitize(membercontact.street_address),
            'POSTI'     : tex_sanitize("%s %s" % (membercontact.postal_code, membercontact.post_office)),
            'BARCODE'   : barcode_4(settings.IBAN_ACCOUNT_NUMBER,cycle.reference_number,None,cycle.sum)
        }
        datalist.append(data)
    return datalist

def generate_pdf(latexfile):
    # TODO: with_timeouting_process(...)
    # set umask to 0077
    oldumask = os.umask(63)
    penv = os.environ.copy()
    pid = Popen(['pdflatex', '-interaction=batchmode', '-output-directory=%s' % TMPDIR, 
        '-no-file-line-error','-halt-on-error','-output-format','pdf',latexfile],
        env=penv, cwd=os.path.dirname(os.path.realpath(settings.PAPER_REMINDER_TEMPLATE)))
    # wait until process ends
    pid.wait()
    try:
        os.remove(latexfile)
    except OSError as e:
        logging.exception(e)
{'fraction_non_alphanumeric': 0.11575035523007979, 'fraction_numerical': 0.00404415783145699, 'mean_word_length': 2.8332635106828654, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2552719', 'n_tokens_mistral': 2970, 'n_tokens_neox': 2856, 'n_words': 649}