<commit_before>import csv
import datetime
import logging
import ntpath
import os
import random
import re
import string
import traceback
from urllib.parse import urljoin

from django import template
from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.auth.models import Group
from django.core.cache import cache
from django.core.exceptions import ObjectDoesNotExist, ImproperlyConfigured
from django.core.files.storage import DefaultStorage
from django.core.urlresolvers import reverse as reverse_djangocore
from django.db.models import Count
from django.template import defaulttags
from matplotlib.backends.backend_svg import FigureCanvasSVG as FigureCanvas
from matplotlib.figure import Figure
from six import StringIO, iteritems

import comicsite.views
from comicmodels.models import ComicSite
from comicsite.core.exceptions import ParserException, PathResolutionException
from comicsite.core.urlresolvers import reverse
from comicsite.templatetags import library_plus
from comicsite.utils.html import escape_for_html_id
from dataproviders.ProjectExcelReader import ProjectExcelReader
from dataproviders.utils.HtmlLinkReplacer import HtmlLinkReplacer
from profiles.models import UserProfile

register = library_plus.LibraryPlus()

logger = logging.getLogger("django")


def parseKeyValueToken(token):
    """Parses token content string into a parameter dictionary
    
    Args:        
        token (django.base.Token): Object representing the string content of
            the template tag. Key values are expected to be of the format         
            key1:value1 key2:value2,...

    Returns:
        A dictionary of key:value pairs
        
    Raises:
        ValueError: if token contents are not in key:val1 key:val2 .. format
        
    """
    split = token.split_contents()
    tag = split[0]
    args = split[1:]

    if "=" in "".join(args):
        raise ValueError(
            "Please use colon ':' instead of equals '=' to separate keys and values")

    return dict([param.split(":") for param in args])


def get_usagestr(function_name):
    """
    Return usage string for a registered template tag function. For displaying
    this info in errors or tag overviews
    """

    if function_name in register.usagestrings:
        usagestr = register.usagestrings[function_name]
    else:
        usagestr = ""

    return sanitize_django_items(usagestr)


@register.tag(name="taglist",
              usagestr="""
              <% taglist %> :
              show all available tags
                       """
              )
def get_taglist(parser, token):
    return TagListNode()


def subdomain_is_projectname():
    """ Check whether this setting is true in settings. Return false if not found

    """

    is_projectname = False

    if hasattr(settings, "SUBDOMAIN_IS_PROJECTNAME"):
        is_projectname = settings.SUBDOMAIN_IS_PROJECTNAME
        if is_projectname and not hasattr(settings, "MAIN_HOST_NAME"):
            msg = """Key 'SUBDOMAIN_IS_PROJECTNAME' was defined in settings,
             but 'MAIN_HOST_NAME' was not. These belong together. Please
             add 'MAIN_HOST_NAME' and set it to the hostname of your site."""
            raise ImproperlyConfigured(msg)

    return is_projectname


@register.tag
def url(parser, token):
    """Overwrites built in url tag to use . It works identicaly, except that where possible
    it will use subdomains to refer to a project instead of a full url path.

    For example, if the subdomain is vessel12.domain.com it will refer to a page
    'details' as /details/ instead of /site/vessel12/details/

    REQUIREMENTS:
    * MIDDLEWARE_CLASSES in settings should contain
      'comicsite.middleware.subdomain.SubdomainMiddleware'

    * These keys should be in the django settings file:
      SUBDOMAIN_IS_PROJECTNAME = True
      MAIN_HOST_NAME = <your site's hostname>

    * APACHE url rewriting should be in effect to rewrite subdomain to
      site/project/. To get you started: the following apache config does this
      for the domain 'devcomicframework.org'
      (put this in your apache config file)

        RewriteEngine   on
        RewriteCond $1 .*/$
        RewriteCond $1 !^/site/.*
        RewriteCond %{HTTP_HOST} !^devcomicframework\.org$
        RewriteCond %{HTTP_HOST} !^www.devcomicframework\.org$
        RewriteCond %{HTTP_HOST} ^([^.]+)\.devcomicframework\.org$
        RewriteRule (.*) /site/%1$1 [PT]


    TODO: turn on and off this behaviour in settings, maybe explicitly define
    base domain to also make it possible to use dots in the base domain.

    """

    orgnode = defaulttags.url(parser, token)
    return comic_URLNode(orgnode.view_name, orgnode.args, orgnode.kwargs,
                         orgnode.asvar)


def filter_by_extension(filenames, extensions):
    """Takes two lists of strings. Return only strings that end with any of 
    the strings in extensions. 
    
    """
    filtered = []
    for extension in extensions:
        filtered = filtered + [f for f in filenames if f.endswith(extension)]
    return filtered


def resolve_path(path, parser, context):
    """Try to resolve all parameters in path   
    
    Paths in COMIC template tag parameters can include variables. Try to
    resolve these and throw error if this is not possible. 
    path can be of three types:
        * a raw filename like "stuff.html" or "results/table1.txt"
        * a filname containing a variable like "results/{{teamid}}/table1.txt"
        * a django template variable like "site.short_name"
    
    Args:
        Path (string)
        parser (django object) 
        context (django context given tag render function)
        
        
    Returns:
        resolved path (string)
    
    Raises:
        PathResolutionException when path cannot be resolved
        :param path: 
                
    """

    # Find out what type it is:

    # If it contains any / or {{ resolving as django var
    # is going to throw an error. Prevent unneeded exception, just skip
    # rendering as var in that case.
    path_resolved = ""
    if not in_list(["{", "}", "\\", "/"], path):
        compiled_filter = parser.compile_filter(strip_quotes(path))
        path_resolved = compiled_filter.resolve(context)

    # if resolved filename is empty, resolution failed, just treat this
    # param as a filepath
    if path_resolved == "":
        filename = strip_quotes(path)
    else:
        filename = path_resolved

    # if there are {{}}'s in there, try to substitute this with url
    # parameter given in the url
    filename = substitute(filename, context["request"].GET.items())

    # If any {{parameters}} are still in filename they were not replaced.
    # This filename is missing information, show this as error text.
    if re.search(r"{{\w+}}", str(filename)):

        missed_parameters = re.findall(r"{{\w+}}", str(filename))
        found_parameters = context["request"].GET.items()

        if not found_parameters:
            found_parameters = "None"
        error_msg = "I am missing required url parameter(s) %s, url parameter(s) found: %s " \
                    "" % (missed_parameters, found_parameters)
        raise PathResolutionException(error_msg)

    return filename


def substitute(string, substitutions):
    """
    Take each key in the substitutions dict. See if this key exists
    between double curly braces in string. If so replace with value.

    Example:
    substitute("my name is {{name}}.",{version:1,name=John})
    > "my name is John"
    """

    for key, value in substitutions:
        string = re.sub("{{" + key + "}}", value, string)

    return string


class comic_URLNode(defaulttags.URLNode):
    def __init__(self, *args, **kwargs):
        super(comic_URLNode, self).__init__(*args, **kwargs)

    def render(self, context):
        # get the url the default django method would give.
        url = super(comic_URLNode, self).render(context)
        url = url.lower()

        if subdomain_is_projectname() and (
            (
                self.view_name.var in [
                    "challenge-homepage",
                    "project_serve_file",
                ]
            ) or (
                self.view_name.var.split(':')[0] in [
                    'evaluation',
                    'teams',
                    'pages',
                    'participants',
                    'admins',
                    'uploads',
                ]
            )
        ):

            # Interpret subdomain as a comicsite. What would normally be the
            # path to this comicsite?

            args = [arg.resolve(context) for arg in self.args]
            project = args[0]

            if project == settings.MAIN_PROJECT_NAME:
                # this url cannot use the domain name shortcut, so it is
                # probably meant as a link the main comicframework site.
                # in that case hardcode the domain to make sure the sub-
                # domain is gone after following this link
                return settings.MAIN_HOST_NAME + url
            else:

                path_to_site = reverse_djangocore("challenge-homepage",
                                                  args=[project]).lower()

                if url.startswith(path_to_site):
                    url = url.replace(path_to_site, "/")

                scheme_subsite_and_host = reverse("challenge-homepage",
                                                  args=[project]).lower()

                return urljoin(scheme_subsite_and_host, url)

        return url


class TagListNode(template.Node):
    """ Print available tags as text
    """

    def __init__(self):
        pass

    def render(self, context):
        html_out = "<table class =\"comictable taglist\">"

        html_out = html_out + "<tr><th>tagname</th><th>description</th></tr>"
        rowclass = "odd"
        for key, val in iteritems(register.usagestrings):
            if not val == "":
                html_out = html_out + "<tr class=\"%s\"><td>%s</td><td>%s</td></tr>\
                        " % (rowclass, key, sanitize_django_items(val))
                if rowclass == "odd":
                    rowclass = "even"
                else:
                    rowclass = "odd"

        html_out = html_out + "</table>"

        return html_out


def sanitize_django_items(string):
    """
    remove {{,{% and other items which would be rendered as tags by django
    """
    out = string
    out = out.replace("{{", "&#123;&#123;")
    out = out.replace("}}", "&#125;&#125;")
    out = out.replace("{%", "&#123;&#37;")
    out = out.replace("%}", "&#37;&#125;")
    out = out.replace(">", "&#62;")
    out = out.replace("<", "&#60;")
    out = out.replace("\n", "<br/>")
    return out


@register.simple_tag
def metafooterpages():
    """ Get html for links to general pages like 'contact' """
    html_string = ""
    pages = comicsite.views.getPages(settings.MAIN_PROJECT_NAME)
    for p in pages:
        if not p.hidden:
            url = reverse('mainproject-home',
                          kwargs={'page_title': p.title})
            if subdomain_is_projectname():
                url = settings.MAIN_HOST_NAME + url
            # TODO: JM add class=active to the active link
            # See https://getbootstrap.com/docs/3.3/components/#navbar
            html_string += "<li><a class='metaFooterMenuItem' href='%s'>" % url
            html_string += p.display_title == "" and p.title or p.display_title
            html_string += "</a></li>"

    return html_string


@register.simple_tag
def main_page_url():
    """ Gets the url to the main page """
    if settings.SUBDOMAIN_IS_PROJECTNAME:
        return settings.MAIN_HOST_NAME
    else:
        return "/"


@register.tag(name="listdir",
              usagestr="""Tag usage: {% listdir <path>:string  <extensionFilter>:ext1,ext2,ext3 %}

              path: directory relative to this projects dropbox folder to list files from. Do not use leading slash.
              extensionFilter: An include filter to specify the file types which should be displayd in the filebrowser.
              """
              )<commit_msg>TRAC #7494 Fix html escaping<commit_after>import csv
import datetime
import logging
import ntpath
import os
import random
import re
import string
import traceback
from urllib.parse import urljoin

from django import template
from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.auth.models import Group
from django.core.cache import cache
from django.core.exceptions import ObjectDoesNotExist, ImproperlyConfigured
from django.core.files.storage import DefaultStorage
from django.core.urlresolvers import reverse as reverse_djangocore
from django.db.models import Count
from django.template import defaulttags
from django.utils.html import format_html
from matplotlib.backends.backend_svg import FigureCanvasSVG as FigureCanvas
from matplotlib.figure import Figure
from six import StringIO, iteritems

import comicsite.views
from comicmodels.models import ComicSite
from comicsite.core.exceptions import ParserException, PathResolutionException
from comicsite.core.urlresolvers import reverse
from comicsite.templatetags import library_plus
from comicsite.utils.html import escape_for_html_id
from dataproviders.ProjectExcelReader import ProjectExcelReader
from dataproviders.utils.HtmlLinkReplacer import HtmlLinkReplacer
from profiles.models import UserProfile

register = library_plus.LibraryPlus()

logger = logging.getLogger("django")


def parseKeyValueToken(token):
    """Parses token content string into a parameter dictionary
    
    Args:        
        token (django.base.Token): Object representing the string content of
            the template tag. Key values are expected to be of the format         
            key1:value1 key2:value2,...

    Returns:
        A dictionary of key:value pairs
        
    Raises:
        ValueError: if token contents are not in key:val1 key:val2 .. format
        
    """
    split = token.split_contents()
    tag = split[0]
    args = split[1:]

    if "=" in "".join(args):
        raise ValueError(
            "Please use colon ':' instead of equals '=' to separate keys and values")

    return dict([param.split(":") for param in args])


def get_usagestr(function_name):
    """
    Return usage string for a registered template tag function. For displaying
    this info in errors or tag overviews
    """

    if function_name in register.usagestrings:
        usagestr = register.usagestrings[function_name]
    else:
        usagestr = ""

    return sanitize_django_items(usagestr)


@register.tag(name="taglist",
              usagestr="""
              <% taglist %> :
              show all available tags
                       """
              )
def get_taglist(parser, token):
    return TagListNode()


def subdomain_is_projectname():
    """ Check whether this setting is true in settings. Return false if not found

    """

    is_projectname = False

    if hasattr(settings, "SUBDOMAIN_IS_PROJECTNAME"):
        is_projectname = settings.SUBDOMAIN_IS_PROJECTNAME
        if is_projectname and not hasattr(settings, "MAIN_HOST_NAME"):
            msg = """Key 'SUBDOMAIN_IS_PROJECTNAME' was defined in settings,
             but 'MAIN_HOST_NAME' was not. These belong together. Please
             add 'MAIN_HOST_NAME' and set it to the hostname of your site."""
            raise ImproperlyConfigured(msg)

    return is_projectname


@register.tag
def url(parser, token):
    """Overwrites built in url tag to use . It works identicaly, except that where possible
    it will use subdomains to refer to a project instead of a full url path.

    For example, if the subdomain is vessel12.domain.com it will refer to a page
    'details' as /details/ instead of /site/vessel12/details/

    REQUIREMENTS:
    * MIDDLEWARE_CLASSES in settings should contain
      'comicsite.middleware.subdomain.SubdomainMiddleware'

    * These keys should be in the django settings file:
      SUBDOMAIN_IS_PROJECTNAME = True
      MAIN_HOST_NAME = <your site's hostname>

    * APACHE url rewriting should be in effect to rewrite subdomain to
      site/project/. To get you started: the following apache config does this
      for the domain 'devcomicframework.org'
      (put this in your apache config file)

        RewriteEngine   on
        RewriteCond $1 .*/$
        RewriteCond $1 !^/site/.*
        RewriteCond %{HTTP_HOST} !^devcomicframework\.org$
        RewriteCond %{HTTP_HOST} !^www.devcomicframework\.org$
        RewriteCond %{HTTP_HOST} ^([^.]+)\.devcomicframework\.org$
        RewriteRule (.*) /site/%1$1 [PT]


    TODO: turn on and off this behaviour in settings, maybe explicitly define
    base domain to also make it possible to use dots in the base domain.

    """

    orgnode = defaulttags.url(parser, token)
    return comic_URLNode(orgnode.view_name, orgnode.args, orgnode.kwargs,
                         orgnode.asvar)


def filter_by_extension(filenames, extensions):
    """Takes two lists of strings. Return only strings that end with any of 
    the strings in extensions. 
    
    """
    filtered = []
    for extension in extensions:
        filtered = filtered + [f for f in filenames if f.endswith(extension)]
    return filtered


def resolve_path(path, parser, context):
    """Try to resolve all parameters in path   
    
    Paths in COMIC template tag parameters can include variables. Try to
    resolve these and throw error if this is not possible. 
    path can be of three types:
        * a raw filename like "stuff.html" or "results/table1.txt"
        * a filname containing a variable like "results/{{teamid}}/table1.txt"
        * a django template variable like "site.short_name"
    
    Args:
        Path (string)
        parser (django object) 
        context (django context given tag render function)
        
        
    Returns:
        resolved path (string)
    
    Raises:
        PathResolutionException when path cannot be resolved
        :param path: 
                
    """

    # Find out what type it is:

    # If it contains any / or {{ resolving as django var
    # is going to throw an error. Prevent unneeded exception, just skip
    # rendering as var in that case.
    path_resolved = ""
    if not in_list(["{", "}", "\\", "/"], path):
        compiled_filter = parser.compile_filter(strip_quotes(path))
        path_resolved = compiled_filter.resolve(context)

    # if resolved filename is empty, resolution failed, just treat this
    # param as a filepath
    if path_resolved == "":
        filename = strip_quotes(path)
    else:
        filename = path_resolved

    # if there are {{}}'s in there, try to substitute this with url
    # parameter given in the url
    filename = substitute(filename, context["request"].GET.items())

    # If any {{parameters}} are still in filename they were not replaced.
    # This filename is missing information, show this as error text.
    if re.search(r"{{\w+}}", str(filename)):

        missed_parameters = re.findall(r"{{\w+}}", str(filename))
        found_parameters = context["request"].GET.items()

        if not found_parameters:
            found_parameters = "None"
        error_msg = "I am missing required url parameter(s) %s, url parameter(s) found: %s " \
                    "" % (missed_parameters, found_parameters)
        raise PathResolutionException(error_msg)

    return filename


def substitute(string, substitutions):
    """
    Take each key in the substitutions dict. See if this key exists
    between double curly braces in string. If so replace with value.

    Example:
    substitute("my name is {{name}}.",{version:1,name=John})
    > "my name is John"
    """

    for key, value in substitutions:
        string = re.sub("{{" + key + "}}", value, string)

    return string


class comic_URLNode(defaulttags.URLNode):
    def __init__(self, *args, **kwargs):
        super(comic_URLNode, self).__init__(*args, **kwargs)

    def render(self, context):
        # get the url the default django method would give.
        url = super(comic_URLNode, self).render(context)
        url = url.lower()

        if subdomain_is_projectname() and (
            (
                self.view_name.var in [
                    "challenge-homepage",
                    "project_serve_file",
                ]
            ) or (
                self.view_name.var.split(':')[0] in [
                    'evaluation',
                    'teams',
                    'pages',
                    'participants',
                    'admins',
                    'uploads',
                ]
            )
        ):

            # Interpret subdomain as a comicsite. What would normally be the
            # path to this comicsite?

            args = [arg.resolve(context) for arg in self.args]
            project = args[0]

            if project == settings.MAIN_PROJECT_NAME:
                # this url cannot use the domain name shortcut, so it is
                # probably meant as a link the main comicframework site.
                # in that case hardcode the domain to make sure the sub-
                # domain is gone after following this link
                return settings.MAIN_HOST_NAME + url
            else:

                path_to_site = reverse_djangocore("challenge-homepage",
                                                  args=[project]).lower()

                if url.startswith(path_to_site):
                    url = url.replace(path_to_site, "/")

                scheme_subsite_and_host = reverse("challenge-homepage",
                                                  args=[project]).lower()

                return urljoin(scheme_subsite_and_host, url)

        return url


class TagListNode(template.Node):
    """ Print available tags as text
    """

    def __init__(self):
        pass

    def render(self, context):
        html_out = "<table class =\"comictable taglist\">"

        html_out = html_out + "<tr><th>tagname</th><th>description</th></tr>"
        rowclass = "odd"
        for key, val in iteritems(register.usagestrings):
            if not val == "":
                html_out = html_out + "<tr class=\"%s\"><td>%s</td><td>%s</td></tr>\
                        " % (rowclass, key, sanitize_django_items(val))
                if rowclass == "odd":
                    rowclass = "even"
                else:
                    rowclass = "odd"

        html_out = html_out + "</table>"

        return html_out


def sanitize_django_items(string):
    """
    remove {{,{% and other items which would be rendered as tags by django
    """
    out = string
    out = out.replace("{{", "&#123;&#123;")
    out = out.replace("}}", "&#125;&#125;")
    out = out.replace("{%", "&#123;&#37;")
    out = out.replace("%}", "&#37;&#125;")
    out = out.replace(">", "&#62;")
    out = out.replace("<", "&#60;")
    out = out.replace("\n", "<br/>")
    return out


@register.simple_tag
def metafooterpages():
    """ Get html for links to general pages like 'contact' """
    html_string = ""
    pages = comicsite.views.getPages(settings.MAIN_PROJECT_NAME)
    for p in pages:
        if not p.hidden:
            url = reverse('mainproject-home',
                          kwargs={'page_title': p.title})
            if subdomain_is_projectname():
                url = settings.MAIN_HOST_NAME + url
            # TODO: JM add class=active to the active link
            # See https://getbootstrap.com/docs/3.3/components/#navbar
            html_string = format_html(
                "<li><a class='metaFooterMenuItem' href='{}'>{}</a></li>",
                url,
                p.display_title if p.display_title else p.title,
            )

    return html_string


@register.simple_tag
def main_page_url():
    """ Gets the url to the main page """
    if settings.SUBDOMAIN_IS_PROJECTNAME:
        return settings.MAIN_HOST_NAME
    else:
        return "/"


@register.tag(name="listdir",
              usagestr="""Tag usage: {% listdir <path>:string  <extensionFilter>:ext1,ext2,ext3 %}

              path: directory relative to this projects dropbox folder to list files from. Do not use leading slash.
              extensionFilter: An include filter to specify the file types which should be displayd in the filebrowser.
              """
              )
{'fraction_non_alphanumeric': 0.09347737464329392, 'fraction_numerical': 0.0044027721157766, 'mean_word_length': 2.35352016404648, 'pattern_counts': {'":': 8, '<': 51, '<?xml version=': 0, '>': 53, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5621386', 'n_tokens_mistral': 6930, 'n_tokens_neox': 6569, 'n_words': 2176}