<commit_before>from __future__ import absolute_import

from tempfile import NamedTemporaryFile
import re

from django.conf import settings
from django.http import HttpResponse
from django.template.response import TemplateResponse
from django.utils.encoding import smart_str
from django.views.generic import TemplateView

from .utils import content_disposition_filename, pathname2fileurl, wkhtmltopdf


class PDFResponse(HttpResponse):
    """HttpResponse that sets the headers for PDF output."""

    def __init__(self, content, mimetype=None, status=200,
                 content_type=None, filename=None, *args, **kwargs):

        if content_type is None:
            content_type = 'application/pdf'

        super(PDFResponse, self).__init__(content=content,
                                          mimetype=mimetype,
                                          status=status,
                                          content_type=content_type)
        self.set_filename(filename)

    def set_filename(self, filename):
        self.filename = filename
        if filename:
            filename = content_disposition_filename(filename)
            header_content = 'attachment; filename={0}'.format(filename)
            self['Content-Disposition'] = header_content
        else:
            del self['Content-Disposition']


class PDFTemplateResponse(TemplateResponse, PDFResponse):
    """Renders a Template into a PDF using wkhtmltopdf"""

    def __init__(self, request, template, context=None, mimetype=None,
                 status=None, content_type=None, current_app=None,
                 filename=None, header_template=None, footer_template=None,
                 cmd_options=None, override_settings=None,
                 *args, **kwargs):

        super(PDFTemplateResponse, self).__init__(request=request,
                                                  template=template,
                                                  context=context,
                                                  mimetype=mimetype,
                                                  status=status,
                                                  content_type=content_type,
                                                  current_app=None,
                                                  *args, **kwargs)
        self.set_filename(filename)

        self.header_template = header_template
        self.footer_template = footer_template

        if cmd_options is None:
            cmd_options = {}
        self.cmd_options = cmd_options

    def render_to_temporary_file(self, template_name, mode='w+b', bufsize=-1,
                                 suffix='.html', prefix='tmp', dir=None,
                                 delete=True):
        template = self.resolve_template(template_name)

        context = self.resolve_context(self.context_data)
        content = smart_str(template.render(context))

        content = self.make_absolute_paths(content)

        tempfile = NamedTemporaryFile(mode=mode, bufsize=bufsize,
                                      suffix=suffix, prefix=prefix,
                                      dir=dir, delete=delete)

        try:
            tempfile.write(content)
            tempfile.flush()
            return tempfile
        except:
            # Clean-up tempfile if an Exception is raised.
            tempfile.close()
            raise

    def convert_to_pdf(self, filename,
                       header_filename=None, footer_filename=None):
        cmd_options = self.cmd_options.copy()
        # Clobber header_html and footer_html only if filenames are
        # provided. These keys may be in self.cmd_options as hardcoded
        # static files.
        if header_filename is not None:
            cmd_options['header_html'] = header_filename
        if footer_filename is not None:
            cmd_options['footer_html'] = footer_filename
        return wkhtmltopdf(pages=[filename], **cmd_options)

    @property
    def rendered_content(self):
        """Returns the freshly rendered content for the template and context
        described by the PDFResponse.

        This *does not* set the final content of the response. To set the
        response content, you must either call render(), or set the
        content explicitly using the value of this property.
        """
        debug = getattr(settings, 'WKHTMLTOPDF_DEBUG', settings.DEBUG)

        input_file = header_file = footer_file = None
        header_filename = footer_filename = None

        try:
            input_file = self.render_to_temporary_file(
                template_name=self.template_name,
                prefix='wkhtmltopdf', suffix='.html',
                delete=(not debug)
            )

            if self.header_template:
                header_file = self.render_to_temporary_file(
                    template_name=self.header_template,
                    prefix='wkhtmltopdf', suffix='.html',
                    delete=(not debug)
                )
                header_filename = header_file.name

            if self.footer_template:
                footer_file = self.render_to_temporary_file(
                    template_name=self.footer_template,
                    prefix='wkhtmltopdf', suffix='.html',
                    delete=(not debug)
                )
                footer_filename = footer_file.name

            return self.convert_to_pdf(filename=input_file.name,
                                       header_filename=header_filename,
                                       footer_filename=footer_filename)
        finally:
            # Clean up temporary files
            for f in filter(None, (input_file, header_file, footer_file)):
                f.close()

    def make_absolute_paths(self, content):
        """Convert all MEDIA files into a file://URL paths in order to correctly get it displayed in PDFs

        mattl's disclaimer: I know it sucks, but it works and I haz no time for better solution now
        """

        overrides = [
            {
            'root': settings.MEDIA_ROOT,
            'url': settings.MEDIA_URL,
            },
            {
            'root': settings.STATIC_ROOT,
            'url': settings.STATIC_URL,
            }
        ]
        has_scheme = re.compile(r'^[^:/]+://')

        for x in overrides:
            if has_scheme.match(x['url']):
                continue

            if not x['root'].endswith('/'):
                x['root'] += '/'

            for occur in re.findall('''["|']({0}.*?)["|']'''.format(x['url']), content):
                content = content.replace(occur, pathname2fileurl(x['root']) + occur[len(x['url']):])

        return content

<commit_msg>Remove duplicates when replacing file paths<commit_after>from __future__ import absolute_import

from tempfile import NamedTemporaryFile
import re

from django.conf import settings
from django.http import HttpResponse
from django.template.response import TemplateResponse
from django.utils.encoding import smart_str
from django.views.generic import TemplateView

from .utils import content_disposition_filename, pathname2fileurl, wkhtmltopdf


class PDFResponse(HttpResponse):
    """HttpResponse that sets the headers for PDF output."""

    def __init__(self, content, mimetype=None, status=200,
                 content_type=None, filename=None, *args, **kwargs):

        if content_type is None:
            content_type = 'application/pdf'

        super(PDFResponse, self).__init__(content=content,
                                          mimetype=mimetype,
                                          status=status,
                                          content_type=content_type)
        self.set_filename(filename)

    def set_filename(self, filename):
        self.filename = filename
        if filename:
            filename = content_disposition_filename(filename)
            header_content = 'attachment; filename={0}'.format(filename)
            self['Content-Disposition'] = header_content
        else:
            del self['Content-Disposition']


class PDFTemplateResponse(TemplateResponse, PDFResponse):
    """Renders a Template into a PDF using wkhtmltopdf"""

    def __init__(self, request, template, context=None, mimetype=None,
                 status=None, content_type=None, current_app=None,
                 filename=None, header_template=None, footer_template=None,
                 cmd_options=None, override_settings=None,
                 *args, **kwargs):

        super(PDFTemplateResponse, self).__init__(request=request,
                                                  template=template,
                                                  context=context,
                                                  mimetype=mimetype,
                                                  status=status,
                                                  content_type=content_type,
                                                  current_app=None,
                                                  *args, **kwargs)
        self.set_filename(filename)

        self.header_template = header_template
        self.footer_template = footer_template

        if cmd_options is None:
            cmd_options = {}
        self.cmd_options = cmd_options

    def render_to_temporary_file(self, template_name, mode='w+b', bufsize=-1,
                                 suffix='.html', prefix='tmp', dir=None,
                                 delete=True):
        template = self.resolve_template(template_name)

        context = self.resolve_context(self.context_data)
        content = smart_str(template.render(context))

        content = self.make_absolute_paths(content)

        tempfile = NamedTemporaryFile(mode=mode, bufsize=bufsize,
                                      suffix=suffix, prefix=prefix,
                                      dir=dir, delete=delete)

        try:
            tempfile.write(content)
            tempfile.flush()
            return tempfile
        except:
            # Clean-up tempfile if an Exception is raised.
            tempfile.close()
            raise

    def convert_to_pdf(self, filename,
                       header_filename=None, footer_filename=None):
        cmd_options = self.cmd_options.copy()
        # Clobber header_html and footer_html only if filenames are
        # provided. These keys may be in self.cmd_options as hardcoded
        # static files.
        if header_filename is not None:
            cmd_options['header_html'] = header_filename
        if footer_filename is not None:
            cmd_options['footer_html'] = footer_filename
        return wkhtmltopdf(pages=[filename], **cmd_options)

    @property
    def rendered_content(self):
        """Returns the freshly rendered content for the template and context
        described by the PDFResponse.

        This *does not* set the final content of the response. To set the
        response content, you must either call render(), or set the
        content explicitly using the value of this property.
        """
        debug = getattr(settings, 'WKHTMLTOPDF_DEBUG', settings.DEBUG)

        input_file = header_file = footer_file = None
        header_filename = footer_filename = None

        try:
            input_file = self.render_to_temporary_file(
                template_name=self.template_name,
                prefix='wkhtmltopdf', suffix='.html',
                delete=(not debug)
            )

            if self.header_template:
                header_file = self.render_to_temporary_file(
                    template_name=self.header_template,
                    prefix='wkhtmltopdf', suffix='.html',
                    delete=(not debug)
                )
                header_filename = header_file.name

            if self.footer_template:
                footer_file = self.render_to_temporary_file(
                    template_name=self.footer_template,
                    prefix='wkhtmltopdf', suffix='.html',
                    delete=(not debug)
                )
                footer_filename = footer_file.name

            return self.convert_to_pdf(filename=input_file.name,
                                       header_filename=header_filename,
                                       footer_filename=footer_filename)
        finally:
            # Clean up temporary files
            for f in filter(None, (input_file, header_file, footer_file)):
                f.close()

    def make_absolute_paths(self, content):
        """Convert all MEDIA files into a file://URL paths in order to correctly get it displayed in PDFs

        mattl's disclaimer: I know it sucks, but it works and I haz no time for better solution now
        """

        overrides = [
            {
            'root': settings.MEDIA_ROOT,
            'url': settings.MEDIA_URL,
            },
            {
            'root': settings.STATIC_ROOT,
            'url': settings.STATIC_URL,
            }
        ]
        has_scheme = re.compile(r'^[^:/]+://')

        for x in overrides:
            if has_scheme.match(x['url']):
                continue

            if not x['root'].endswith('/'):
                x['root'] += '/'

            occurences = re.findall('''["|']({0}.*?)["|']'''.format(x['url']),
                                    content)
            occurences = list(set(occurences))  # Remove dups
            for occur in occurences:
                content = content.replace(occur,
                                          pathname2fileurl(x['root']) +
                                          occur[len(x['url']):])

        return content


{'fraction_non_alphanumeric': 0.08056490560515148, 'fraction_numerical': 0.0011707888189667788, 'mean_word_length': 1.677703761755486, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 57, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3670029', 'n_tokens_mistral': 3365, 'n_tokens_neox': 3188, 'n_words': 795}